mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Dragan Spiridonov
d95f0d7b18
The cross-model consistency matrix (models_to_test) in 54 skill evals + both eval templates pinned retired/malformed model IDs — `claude-3.5-sonnet` (malformed: dashes, not dots; resolves to Sonnet 3.5, retired Oct 2025) and `claude-3-haiku` (retired Apr 19 2026). Both 404, so those evals could not run a meaningful cross-model check. Refresh to current pinned IDs, preserving the original "primary + fast floor" design: - claude-sonnet-4-6 # primary (high accuracy) - claude-haiku-4-5 # fast floor (minimum quality) High-stakes skills additionally get the capability ceiling claude-opus-4-8 (security-testing, security-visual-testing, n8n-security-testing, pentest-validation, chaos-engineering-resilience, qe-chaos-resilience, qe-defect-intelligence) and the security eval template. Opus is scoped to these 7 + the security template rather than all 54 to avoid a fleet-wide eval-cost blowup. Also fixes the two `--model claude-3.5-sonnet/claude-3-haiku` examples in run-skill-eval.ts and a nested cross-model test-case input in qe-learning-optimization.yaml. Verified: all eval YAML parses; zero retired/deprecated Claude IDs remain across skills/evals + templates + runner. (src/domains/constants.ts MODEL_TIERS was already current — tier 1 claude-haiku-4-5 — so it is intentionally unchanged; the legacy claude-3-haiku-20240307 entries in cost-tracker/bedrock are retained historical pricing/availability data.) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1298 lines
42 KiB
TypeScript
1298 lines
42 KiB
TypeScript
#!/usr/bin/env npx tsx
|
|
/**
|
|
* AQE Skill Evaluation Runner
|
|
*
|
|
* Runs skill evaluation test suites defined in YAML format.
|
|
* Integrates with AQE MCP tools for shared learning and QualityFeedbackLoop.
|
|
*
|
|
* Usage:
|
|
* npx tsx scripts/run-skill-eval.ts --skill security-testing --model claude-sonnet-4-6
|
|
* npx tsx scripts/run-skill-eval.ts --eval-file .claude/skills/security-testing/evals/security-testing.yaml
|
|
* npx tsx scripts/run-skill-eval.ts --skill security-testing --all-models
|
|
*
|
|
* MCP Integration (per docs/specs/skill-validation-mcp-integration.md):
|
|
* - Queries patterns before running: mcp__agentic-qe__memory_query
|
|
* - Tracks test outcomes: mcp__agentic-qe__test_outcome_track
|
|
* - Stores patterns after: mcp__agentic-qe__memory_store
|
|
* - Shares learning: mcp__agentic-qe__memory_share
|
|
* - Updates quality gate: mcp__agentic-qe__quality_assess
|
|
*
|
|
* @module scripts/run-skill-eval
|
|
* @version 1.0.0
|
|
*/
|
|
|
|
import { readFileSync, writeFileSync, existsSync } from 'fs';
|
|
import { join, dirname, basename } from 'path';
|
|
import { fileURLToPath } from 'url';
|
|
|
|
// For YAML parsing - using dynamic import for ESM compatibility
|
|
let yaml: typeof import('yaml');
|
|
|
|
// =============================================================================
|
|
// Types
|
|
// =============================================================================
|
|
|
|
/**
|
|
* Test case from eval YAML
|
|
*/
|
|
interface TestCase {
|
|
id: string;
|
|
description: string;
|
|
category?: string;
|
|
priority: 'critical' | 'high' | 'medium' | 'low';
|
|
skip?: boolean;
|
|
skip_reason?: string;
|
|
negative_control?: boolean;
|
|
input: TestInput;
|
|
expected_output: ExpectedOutput;
|
|
validation?: ValidationConfig;
|
|
timeout_ms?: number;
|
|
}
|
|
|
|
interface TestInput {
|
|
code?: string;
|
|
file_path?: string;
|
|
url?: string;
|
|
prompt?: string;
|
|
context?: {
|
|
language?: string;
|
|
framework?: string;
|
|
environment?: 'development' | 'staging' | 'production';
|
|
};
|
|
options?: Record<string, unknown>;
|
|
}
|
|
|
|
interface ExpectedOutput {
|
|
must_contain?: string[];
|
|
must_not_contain?: string[];
|
|
must_match_regex?: string[];
|
|
severity_classification?: 'critical' | 'high' | 'medium' | 'low' | 'info';
|
|
finding_count?: { min?: number; max?: number };
|
|
recommendation_count?: { min?: number; max?: number };
|
|
schema_path?: string;
|
|
custom_assertions?: CustomAssertion[];
|
|
}
|
|
|
|
interface CustomAssertion {
|
|
type: 'jsonpath' | 'semantic' | 'function';
|
|
expression: string;
|
|
expected?: unknown;
|
|
}
|
|
|
|
interface ValidationConfig {
|
|
schema_check?: boolean;
|
|
keyword_match_threshold?: number;
|
|
reasoning_quality_min?: number;
|
|
semantic_similarity_min?: number;
|
|
allow_partial?: boolean;
|
|
adaptive_rubric?: boolean;
|
|
grading_rubric?: {
|
|
completeness?: number;
|
|
accuracy?: number;
|
|
actionability?: number;
|
|
};
|
|
}
|
|
|
|
interface MCPIntegrationConfig {
|
|
enabled: boolean;
|
|
namespace: string;
|
|
store_patterns: boolean;
|
|
query_patterns: boolean;
|
|
track_outcomes: boolean;
|
|
share_learning: boolean;
|
|
update_quality_gate: boolean;
|
|
target_agents: string[];
|
|
}
|
|
|
|
interface LearningConfig {
|
|
store_success_patterns: boolean;
|
|
store_failure_patterns: boolean;
|
|
pattern_ttl_days: number;
|
|
min_confidence_to_store: number;
|
|
cross_model_comparison: boolean;
|
|
}
|
|
|
|
interface SuccessCriteria {
|
|
pass_rate: number;
|
|
critical_pass_rate?: number;
|
|
avg_reasoning_quality?: number;
|
|
max_execution_time_ms?: number;
|
|
cross_model_variance?: number;
|
|
}
|
|
|
|
/**
|
|
* Full eval suite configuration
|
|
*/
|
|
interface EvalSuite {
|
|
skill: string;
|
|
version: string;
|
|
description?: string;
|
|
models_to_test: string[];
|
|
mcp_integration?: MCPIntegrationConfig;
|
|
learning?: LearningConfig;
|
|
setup?: {
|
|
required_tools?: string[];
|
|
environment_variables?: Record<string, string>;
|
|
fixtures?: Array<{ name: string; path: string; content: string }>;
|
|
};
|
|
test_cases: TestCase[];
|
|
success_criteria: SuccessCriteria;
|
|
metadata?: {
|
|
author?: string;
|
|
created?: string;
|
|
last_updated?: string;
|
|
coverage_target?: string;
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Result of a single test case execution
|
|
*/
|
|
interface TestCaseResult {
|
|
id: string;
|
|
description: string;
|
|
category?: string;
|
|
priority: string;
|
|
passed: boolean;
|
|
skipped: boolean;
|
|
skip_reason?: string;
|
|
execution_time_ms: number;
|
|
keyword_match_score: number;
|
|
reasoning_quality_score: number;
|
|
validation_details: {
|
|
must_contain_matches: string[];
|
|
must_contain_misses: string[];
|
|
must_not_contain_violations: string[];
|
|
regex_matches: string[];
|
|
regex_misses: string[];
|
|
severity_matched: boolean;
|
|
finding_count_matched: boolean;
|
|
negative_control_passed?: boolean;
|
|
finding_count_actual?: number;
|
|
schema_validation_passed?: boolean;
|
|
rubric_breakdown?: {
|
|
completeness: number;
|
|
accuracy: number;
|
|
actionability: number;
|
|
weighted_score: number;
|
|
};
|
|
adaptive_keywords_extracted?: string[];
|
|
};
|
|
raw_output?: string;
|
|
error?: string;
|
|
}
|
|
|
|
/**
|
|
* Result of running eval suite against a single model
|
|
*/
|
|
interface ModelEvalResult {
|
|
model: string;
|
|
skill: string;
|
|
version: string;
|
|
timestamp: string;
|
|
total_tests: number;
|
|
passed: number;
|
|
failed: number;
|
|
skipped: number;
|
|
pass_rate: number;
|
|
critical_pass_rate: number;
|
|
avg_reasoning_quality: number;
|
|
total_execution_time_ms: number;
|
|
test_results: TestCaseResult[];
|
|
success_criteria_met: boolean;
|
|
criteria_failures: string[];
|
|
}
|
|
|
|
/**
|
|
* Aggregated result across all models
|
|
*/
|
|
interface EvalRunResult {
|
|
skill: string;
|
|
version: string;
|
|
run_id: string;
|
|
timestamp: string;
|
|
models_tested: string[];
|
|
model_results: ModelEvalResult[];
|
|
cross_model_variance: number;
|
|
overall_passed: boolean;
|
|
summary: {
|
|
best_model: string;
|
|
worst_model: string;
|
|
avg_pass_rate: number;
|
|
recommendations: string[];
|
|
};
|
|
mcp_integration_log: MCPIntegrationLog;
|
|
}
|
|
|
|
/**
|
|
* Log of MCP tool calls made during evaluation
|
|
*/
|
|
interface MCPIntegrationLog {
|
|
patterns_queried: number;
|
|
outcomes_tracked: number;
|
|
patterns_stored: number;
|
|
learning_shared: boolean;
|
|
quality_gate_updated: boolean;
|
|
errors: string[];
|
|
}
|
|
|
|
// =============================================================================
|
|
// MCP Integration Layer
|
|
// =============================================================================
|
|
|
|
/**
|
|
* Mock MCP client for demonstration
|
|
* In production, this would connect to the actual AQE MCP server
|
|
*/
|
|
class MCPClient {
|
|
private enabled: boolean;
|
|
private namespace: string;
|
|
private log: MCPIntegrationLog;
|
|
|
|
constructor(config?: MCPIntegrationConfig) {
|
|
this.enabled = config?.enabled ?? false;
|
|
this.namespace = config?.namespace ?? 'skill-validation';
|
|
this.log = {
|
|
patterns_queried: 0,
|
|
outcomes_tracked: 0,
|
|
patterns_stored: 0,
|
|
learning_shared: false,
|
|
quality_gate_updated: false,
|
|
errors: [],
|
|
};
|
|
}
|
|
|
|
getLog(): MCPIntegrationLog {
|
|
return { ...this.log };
|
|
}
|
|
|
|
/**
|
|
* Query existing patterns before running evals
|
|
* MCP Tool: mcp__agentic-qe__memory_query
|
|
*/
|
|
async queryPatterns(skill: string): Promise<unknown[]> {
|
|
if (!this.enabled) return [];
|
|
|
|
console.log(`[MCP] Querying patterns for skill: ${skill}`);
|
|
|
|
// In production, this would call:
|
|
// await mcp__agentic-qe__memory_query({
|
|
// pattern: `skill-validation-${skill}-*`,
|
|
// namespace: this.namespace,
|
|
// limit: 10
|
|
// });
|
|
|
|
this.log.patterns_queried++;
|
|
|
|
// Return mock patterns for demonstration
|
|
return [
|
|
{
|
|
patternId: `${skill}-baseline-pattern`,
|
|
successRate: 0.92,
|
|
lastValidated: new Date().toISOString(),
|
|
},
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Track individual test outcome
|
|
* MCP Tool: mcp__agentic-qe__test_outcome_track
|
|
*/
|
|
async trackOutcome(
|
|
testId: string,
|
|
passed: boolean,
|
|
patternId?: string,
|
|
coverage?: { lines: number; branches: number; functions: number },
|
|
executionTime?: number
|
|
): Promise<void> {
|
|
if (!this.enabled) return;
|
|
|
|
console.log(`[MCP] Tracking outcome: ${testId} - ${passed ? 'PASSED' : 'FAILED'}`);
|
|
|
|
// In production, this would call:
|
|
// await mcp__agentic-qe__test_outcome_track({
|
|
// testId: `skill-${skill}-${testId}`,
|
|
// generatedBy: 'eval-runner',
|
|
// patternId,
|
|
// passed,
|
|
// coverage,
|
|
// executionTime,
|
|
// flaky: false
|
|
// });
|
|
|
|
this.log.outcomes_tracked++;
|
|
}
|
|
|
|
/**
|
|
* Store successful patterns for future reference
|
|
* MCP Tool: mcp__agentic-qe__memory_store
|
|
*/
|
|
async storePattern(
|
|
skill: string,
|
|
results: ModelEvalResult,
|
|
patterns: unknown[]
|
|
): Promise<void> {
|
|
if (!this.enabled) return;
|
|
|
|
console.log(`[MCP] Storing patterns for skill: ${skill}`);
|
|
|
|
const key = `skill-validation-${skill}-${Date.now()}`;
|
|
const value = {
|
|
skillName: skill,
|
|
trustTier: 3, // Level 3 = has eval suite
|
|
validationResult: {
|
|
passRate: results.pass_rate,
|
|
criticalPassRate: results.critical_pass_rate,
|
|
avgReasoningQuality: results.avg_reasoning_quality,
|
|
},
|
|
model: results.model,
|
|
passRate: results.pass_rate,
|
|
patterns,
|
|
timestamp: new Date().toISOString(),
|
|
};
|
|
|
|
// In production, this would call:
|
|
// await mcp__agentic-qe__memory_store({
|
|
// key,
|
|
// value,
|
|
// namespace: this.namespace
|
|
// });
|
|
|
|
this.log.patterns_stored++;
|
|
}
|
|
|
|
/**
|
|
* Share learning with fleet agents
|
|
* MCP Tool: mcp__agentic-qe__memory_share
|
|
*/
|
|
async shareLearning(
|
|
skill: string,
|
|
targetAgents: string[],
|
|
insights: unknown
|
|
): Promise<void> {
|
|
if (!this.enabled) return;
|
|
|
|
console.log(`[MCP] Sharing learning with agents: ${targetAgents.join(', ')}`);
|
|
|
|
// In production, this would call:
|
|
// await mcp__agentic-qe__memory_share({
|
|
// sourceAgentId: 'eval-runner',
|
|
// targetAgentIds: targetAgents,
|
|
// knowledgeDomain: 'skill-validation',
|
|
// data: {
|
|
// skillName: skill,
|
|
// insights,
|
|
// recommendations: []
|
|
// }
|
|
// });
|
|
|
|
this.log.learning_shared = true;
|
|
}
|
|
|
|
/**
|
|
* Update quality gate with validation metrics
|
|
* MCP Tool: mcp__agentic-qe__quality_assess
|
|
*/
|
|
async updateQualityGate(skill: string, metrics: unknown): Promise<void> {
|
|
if (!this.enabled) return;
|
|
|
|
console.log(`[MCP] Updating quality gate for skill: ${skill}`);
|
|
|
|
// In production, this would call:
|
|
// await mcp__agentic-qe__quality_assess({
|
|
// target: `skill:${skill}`,
|
|
// metrics,
|
|
// updateQualityScore: true
|
|
// });
|
|
|
|
this.log.quality_gate_updated = true;
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Evaluation Engine
|
|
// =============================================================================
|
|
|
|
class SkillEvaluationRunner {
|
|
private suite: EvalSuite;
|
|
private mcpClient: MCPClient;
|
|
private verbose: boolean;
|
|
|
|
constructor(suite: EvalSuite, verbose = false) {
|
|
this.suite = suite;
|
|
this.verbose = verbose;
|
|
this.mcpClient = new MCPClient(suite.mcp_integration);
|
|
}
|
|
|
|
/**
|
|
* Run evaluation against a single model
|
|
*/
|
|
async runForModel(model: string): Promise<ModelEvalResult> {
|
|
const startTime = Date.now();
|
|
const results: TestCaseResult[] = [];
|
|
|
|
console.log(`\n${'='.repeat(60)}`);
|
|
console.log(`Running ${this.suite.skill} evals against ${model}`);
|
|
console.log(`${'='.repeat(60)}\n`);
|
|
|
|
// Query existing patterns (MCP integration)
|
|
await this.mcpClient.queryPatterns(this.suite.skill);
|
|
|
|
// Run each test case
|
|
for (const testCase of this.suite.test_cases) {
|
|
const result = await this.runTestCase(testCase, model);
|
|
results.push(result);
|
|
|
|
// Track outcome (MCP integration)
|
|
await this.mcpClient.trackOutcome(
|
|
testCase.id,
|
|
result.passed,
|
|
undefined,
|
|
undefined,
|
|
result.execution_time_ms
|
|
);
|
|
|
|
// Log progress
|
|
const status = result.skipped ? 'SKIP' : result.passed ? 'PASS' : 'FAIL';
|
|
const icon = result.skipped ? '-' : result.passed ? '+' : 'x';
|
|
console.log(` [${icon}] ${testCase.id}: ${status} (${result.execution_time_ms}ms)`);
|
|
}
|
|
|
|
const totalTime = Date.now() - startTime;
|
|
|
|
// Calculate metrics
|
|
const passed = results.filter((r) => r.passed && !r.skipped).length;
|
|
const failed = results.filter((r) => !r.passed && !r.skipped).length;
|
|
const skipped = results.filter((r) => r.skipped).length;
|
|
const total = results.length - skipped;
|
|
|
|
const criticalTests = results.filter(
|
|
(r) => this.suite.test_cases.find((tc) => tc.id === r.id)?.priority === 'critical'
|
|
);
|
|
const criticalPassed = criticalTests.filter((r) => r.passed && !r.skipped).length;
|
|
const criticalTotal = criticalTests.filter((r) => !r.skipped).length;
|
|
|
|
const avgReasoningQuality =
|
|
results
|
|
.filter((r) => !r.skipped)
|
|
.reduce((sum, r) => sum + r.reasoning_quality_score, 0) / total || 0;
|
|
|
|
// Check success criteria
|
|
const criteriaFailures: string[] = [];
|
|
const passRate = total > 0 ? passed / total : 0;
|
|
const criticalPassRate = criticalTotal > 0 ? criticalPassed / criticalTotal : 1;
|
|
|
|
if (passRate < this.suite.success_criteria.pass_rate) {
|
|
criteriaFailures.push(
|
|
`Pass rate ${(passRate * 100).toFixed(1)}% < required ${(this.suite.success_criteria.pass_rate * 100).toFixed(1)}%`
|
|
);
|
|
}
|
|
|
|
if (
|
|
this.suite.success_criteria.critical_pass_rate &&
|
|
criticalPassRate < this.suite.success_criteria.critical_pass_rate
|
|
) {
|
|
criteriaFailures.push(
|
|
`Critical pass rate ${(criticalPassRate * 100).toFixed(1)}% < required ${(this.suite.success_criteria.critical_pass_rate * 100).toFixed(1)}%`
|
|
);
|
|
}
|
|
|
|
if (
|
|
this.suite.success_criteria.avg_reasoning_quality &&
|
|
avgReasoningQuality < this.suite.success_criteria.avg_reasoning_quality
|
|
) {
|
|
criteriaFailures.push(
|
|
`Avg reasoning quality ${avgReasoningQuality.toFixed(2)} < required ${this.suite.success_criteria.avg_reasoning_quality}`
|
|
);
|
|
}
|
|
|
|
if (
|
|
this.suite.success_criteria.max_execution_time_ms &&
|
|
totalTime > this.suite.success_criteria.max_execution_time_ms
|
|
) {
|
|
criteriaFailures.push(
|
|
`Execution time ${totalTime}ms > max ${this.suite.success_criteria.max_execution_time_ms}ms`
|
|
);
|
|
}
|
|
|
|
const modelResult: ModelEvalResult = {
|
|
model,
|
|
skill: this.suite.skill,
|
|
version: this.suite.version,
|
|
timestamp: new Date().toISOString(),
|
|
total_tests: results.length,
|
|
passed,
|
|
failed,
|
|
skipped,
|
|
pass_rate: passRate,
|
|
critical_pass_rate: criticalPassRate,
|
|
avg_reasoning_quality: avgReasoningQuality,
|
|
total_execution_time_ms: totalTime,
|
|
test_results: results,
|
|
success_criteria_met: criteriaFailures.length === 0,
|
|
criteria_failures: criteriaFailures,
|
|
};
|
|
|
|
// Store patterns (MCP integration)
|
|
if (this.suite.mcp_integration?.store_patterns) {
|
|
const learnedPatterns = this.extractPatterns(results);
|
|
await this.mcpClient.storePattern(this.suite.skill, modelResult, learnedPatterns);
|
|
}
|
|
|
|
return modelResult;
|
|
}
|
|
|
|
/**
|
|
* Run a single test case
|
|
*/
|
|
private async runTestCase(testCase: TestCase, model: string): Promise<TestCaseResult> {
|
|
const startTime = Date.now();
|
|
|
|
// Handle skipped tests
|
|
if (testCase.skip) {
|
|
return {
|
|
id: testCase.id,
|
|
description: testCase.description,
|
|
category: testCase.category,
|
|
priority: testCase.priority,
|
|
passed: false,
|
|
skipped: true,
|
|
skip_reason: testCase.skip_reason,
|
|
execution_time_ms: 0,
|
|
keyword_match_score: 0,
|
|
reasoning_quality_score: 0,
|
|
validation_details: {
|
|
must_contain_matches: [],
|
|
must_contain_misses: [],
|
|
must_not_contain_violations: [],
|
|
regex_matches: [],
|
|
regex_misses: [],
|
|
severity_matched: false,
|
|
finding_count_matched: false,
|
|
},
|
|
};
|
|
}
|
|
|
|
try {
|
|
// In production, this would invoke the actual skill via Claude API
|
|
// For now, we simulate the output based on test case expectations
|
|
const output = await this.simulateSkillExecution(testCase, model);
|
|
const executionTime = Date.now() - startTime;
|
|
|
|
// Validate output against expectations
|
|
const validation = this.validateOutput(output, testCase.expected_output, testCase.validation, testCase);
|
|
|
|
return {
|
|
id: testCase.id,
|
|
description: testCase.description,
|
|
category: testCase.category,
|
|
priority: testCase.priority,
|
|
passed: validation.passed,
|
|
skipped: false,
|
|
execution_time_ms: executionTime,
|
|
keyword_match_score: validation.keywordMatchScore,
|
|
reasoning_quality_score: validation.reasoningQualityScore,
|
|
validation_details: validation.details,
|
|
raw_output: this.verbose ? output : undefined,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
id: testCase.id,
|
|
description: testCase.description,
|
|
category: testCase.category,
|
|
priority: testCase.priority,
|
|
passed: false,
|
|
skipped: false,
|
|
execution_time_ms: Date.now() - startTime,
|
|
keyword_match_score: 0,
|
|
reasoning_quality_score: 0,
|
|
validation_details: {
|
|
must_contain_matches: [],
|
|
must_contain_misses: [],
|
|
must_not_contain_violations: [],
|
|
regex_matches: [],
|
|
regex_misses: [],
|
|
severity_matched: false,
|
|
finding_count_matched: false,
|
|
},
|
|
error: error instanceof Error ? error.message : String(error),
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Simulate skill execution (placeholder for actual LLM call)
|
|
* In production, this would:
|
|
* 1. Load the skill from .claude/skills/{skill}/SKILL.md
|
|
* 2. Call the LLM API with the skill prompt and test input
|
|
* 3. Return the actual LLM output
|
|
*/
|
|
private async simulateSkillExecution(testCase: TestCase, model: string): Promise<string> {
|
|
// Simulate some processing time
|
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
|
|
// For demonstration, generate mock output based on test case category
|
|
const category = testCase.category || 'general';
|
|
|
|
// This is a simulation - in production this would be the actual LLM response
|
|
if (category === 'negative') {
|
|
return JSON.stringify({
|
|
findings: [],
|
|
summary: 'The code appears to follow security best practices. No critical vulnerabilities detected.',
|
|
recommendations: ['Continue following current security patterns'],
|
|
severity: 'info',
|
|
});
|
|
}
|
|
|
|
// For injection/auth/crypto tests, return findings
|
|
return JSON.stringify({
|
|
findings: [
|
|
{
|
|
type: testCase.expected_output.must_contain?.[0]?.toLowerCase() || 'vulnerability',
|
|
severity: testCase.expected_output.severity_classification || 'high',
|
|
description: `Detected potential ${testCase.expected_output.must_contain?.[0] || 'security issue'}`,
|
|
cwe: 'CWE-89',
|
|
owasp: 'A03:2021',
|
|
remediation: 'Use parameterized queries and proper input validation',
|
|
},
|
|
],
|
|
summary: `Security analysis complete. Found ${testCase.expected_output.finding_count?.min || 1} issue(s).`,
|
|
recommendations: ['Implement secure coding practices', 'Use prepared statements'],
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Validate LLM output against expected output criteria.
|
|
*
|
|
* Implements:
|
|
* - P1-6: Negative control (inverted pass logic for "should decline" tests)
|
|
* - P1-5a: Finding count enforcement + schema_path validation
|
|
* - P1-5b: Grading rubric weighted scoring (completeness/accuracy/actionability)
|
|
* - P1-4: Adaptive rubric (dynamic keyword extraction from prompt)
|
|
*/
|
|
private validateOutput(
|
|
output: string,
|
|
expected: ExpectedOutput,
|
|
config?: ValidationConfig,
|
|
testCase?: TestCase
|
|
): {
|
|
passed: boolean;
|
|
keywordMatchScore: number;
|
|
reasoningQualityScore: number;
|
|
details: TestCaseResult['validation_details'];
|
|
} {
|
|
const outputLower = output.toLowerCase();
|
|
const threshold = config?.keyword_match_threshold ?? 0.8;
|
|
const isNegativeControl = testCase?.negative_control ?? false;
|
|
|
|
// --- P1-4: Adaptive Rubric — extract additional keywords from prompt ---
|
|
const adaptiveKeywords: string[] = [];
|
|
let effectiveMustContain = expected.must_contain || [];
|
|
|
|
if (config?.adaptive_rubric && testCase?.input?.prompt) {
|
|
const prompt = testCase.input.prompt;
|
|
// Extract quoted strings
|
|
const quoted = prompt.match(/"([^"]+)"/g)?.map((s) => s.replace(/"/g, '')) || [];
|
|
// Extract format keywords
|
|
const formatWords = prompt.match(/\b(JSON|YAML|markdown|XML|CSV|HTML|SQL)\b/gi) || [];
|
|
// Extract named standards
|
|
const standards =
|
|
prompt.match(/\b(OWASP|WCAG|PCI[-\s]DSS|HIPAA|SOC2|GDPR|ISO\s?\d+)\b/gi) || [];
|
|
// Extract meaningful numbers (>1 digit or >5)
|
|
const numbers = (prompt.match(/\b\d+(\.\d+)?\b/g) || []).filter(
|
|
(n) => n.length > 1 || parseInt(n) > 5
|
|
);
|
|
|
|
adaptiveKeywords.push(...quoted, ...formatWords, ...standards, ...numbers);
|
|
|
|
// Merge with existing must_contain (deduplicate)
|
|
const existingLower = new Set(effectiveMustContain.map((k) => k.toLowerCase()));
|
|
const newKeywords = adaptiveKeywords.filter((k) => !existingLower.has(k.toLowerCase()));
|
|
effectiveMustContain = [...effectiveMustContain, ...newKeywords];
|
|
}
|
|
|
|
// Check must_contain
|
|
const mustContainMatches: string[] = [];
|
|
const mustContainMisses: string[] = [];
|
|
|
|
for (const keyword of effectiveMustContain) {
|
|
if (outputLower.includes(keyword.toLowerCase())) {
|
|
mustContainMatches.push(keyword);
|
|
} else {
|
|
mustContainMisses.push(keyword);
|
|
}
|
|
}
|
|
|
|
// Check must_not_contain
|
|
const violations: string[] = [];
|
|
for (const keyword of expected.must_not_contain || []) {
|
|
if (outputLower.includes(keyword.toLowerCase())) {
|
|
violations.push(keyword);
|
|
}
|
|
}
|
|
|
|
// Check regex patterns
|
|
const regexMatches: string[] = [];
|
|
const regexMisses: string[] = [];
|
|
for (const pattern of expected.must_match_regex || []) {
|
|
try {
|
|
const regex = new RegExp(pattern, 'i');
|
|
if (regex.test(output)) {
|
|
regexMatches.push(pattern);
|
|
} else {
|
|
regexMisses.push(pattern);
|
|
}
|
|
} catch {
|
|
regexMisses.push(pattern);
|
|
}
|
|
}
|
|
|
|
// Calculate keyword match score
|
|
const totalKeywords = effectiveMustContain.length + (expected.must_match_regex?.length || 0);
|
|
const matchedKeywords = mustContainMatches.length + regexMatches.length;
|
|
const keywordMatchScore = totalKeywords > 0 ? matchedKeywords / totalKeywords : 1;
|
|
|
|
// Check severity classification
|
|
const severityMatched =
|
|
!expected.severity_classification ||
|
|
outputLower.includes(expected.severity_classification.toLowerCase());
|
|
|
|
// --- P1-5a: Finding count enforcement ---
|
|
let findingCountMatched = true;
|
|
let findingCountActual: number | undefined;
|
|
|
|
if (expected.finding_count) {
|
|
// Try JSON parsing first
|
|
try {
|
|
const parsed = JSON.parse(output);
|
|
if (Array.isArray(parsed?.findings)) {
|
|
findingCountActual = parsed.findings.length;
|
|
}
|
|
} catch {
|
|
// Fallback: count severity-keyword occurrences or numbered findings
|
|
const severityPattern =
|
|
/\b(critical|high|medium|low|info)\b.*?(vulnerability|finding|issue)/gi;
|
|
const numberedPattern = /^\s*\d+[.)]\s+/gm;
|
|
const severityCount = (output.match(severityPattern) || []).length;
|
|
const numberedCount = (output.match(numberedPattern) || []).length;
|
|
findingCountActual = Math.max(severityCount, numberedCount);
|
|
}
|
|
|
|
if (findingCountActual !== undefined) {
|
|
if (
|
|
expected.finding_count.min !== undefined &&
|
|
findingCountActual < expected.finding_count.min
|
|
) {
|
|
findingCountMatched = false;
|
|
}
|
|
if (
|
|
expected.finding_count.max !== undefined &&
|
|
findingCountActual > expected.finding_count.max
|
|
) {
|
|
findingCountMatched = false;
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- P1-5a: Schema validation ---
|
|
let schemaValidationPassed: boolean | undefined;
|
|
if (expected.schema_path) {
|
|
try {
|
|
const schemaContent = readFileSync(expected.schema_path, 'utf-8');
|
|
const schema = JSON.parse(schemaContent);
|
|
const parsed = JSON.parse(output);
|
|
// Structural check: verify all required fields exist
|
|
if (schema.required && Array.isArray(schema.required)) {
|
|
const missingFields = schema.required.filter(
|
|
(field: string) => !(field in parsed)
|
|
);
|
|
schemaValidationPassed = missingFields.length === 0;
|
|
} else {
|
|
schemaValidationPassed = true;
|
|
}
|
|
} catch {
|
|
schemaValidationPassed = false;
|
|
}
|
|
}
|
|
|
|
// --- P1-5b: Grading Rubric scoring ---
|
|
let rubricBreakdown:
|
|
| {
|
|
completeness: number;
|
|
accuracy: number;
|
|
actionability: number;
|
|
weighted_score: number;
|
|
}
|
|
| undefined;
|
|
|
|
if (config?.grading_rubric) {
|
|
const weights = config.grading_rubric;
|
|
const totalWeight =
|
|
(weights.completeness ?? 0) + (weights.accuracy ?? 0) + (weights.actionability ?? 0);
|
|
|
|
// Completeness = fraction of must_contain matched
|
|
const completenessScore = totalKeywords > 0 ? matchedKeywords / totalKeywords : 1;
|
|
|
|
// Accuracy = 1 - (violations / total_must_not_contain)
|
|
const totalMustNotContain = expected.must_not_contain?.length || 0;
|
|
const accuracyScore =
|
|
totalMustNotContain > 0 ? 1 - violations.length / totalMustNotContain : 1;
|
|
|
|
// Actionability = heuristic: code blocks, numbered steps, specific recommendations
|
|
let actionabilityScore = 0;
|
|
const hasCodeBlocks = /```[\s\S]*?```/.test(output) || /`[^`]+`/.test(output);
|
|
const hasNumberedSteps = /^\s*\d+[.)]\s+/m.test(output);
|
|
const hasRecommendations =
|
|
/\b(recommend|suggest|should|must|consider|implement|use|apply|ensure)\b/i.test(output);
|
|
const hasSpecificActions =
|
|
/\b(install|configure|update|replace|add|remove|change|set|enable|disable)\b/i.test(
|
|
output
|
|
);
|
|
|
|
if (hasCodeBlocks) actionabilityScore += 0.3;
|
|
if (hasNumberedSteps) actionabilityScore += 0.3;
|
|
if (hasRecommendations) actionabilityScore += 0.2;
|
|
if (hasSpecificActions) actionabilityScore += 0.2;
|
|
actionabilityScore = Math.min(1, actionabilityScore);
|
|
|
|
// Weighted sum
|
|
const weightedScore =
|
|
totalWeight > 0
|
|
? ((weights.completeness ?? 0) * completenessScore +
|
|
(weights.accuracy ?? 0) * accuracyScore +
|
|
(weights.actionability ?? 0) * actionabilityScore) /
|
|
totalWeight
|
|
: completenessScore;
|
|
|
|
rubricBreakdown = {
|
|
completeness: completenessScore,
|
|
accuracy: accuracyScore,
|
|
actionability: actionabilityScore,
|
|
weighted_score: weightedScore,
|
|
};
|
|
}
|
|
|
|
// Calculate reasoning quality — use rubric if available, else simple heuristic
|
|
const reasoningQualityScore = rubricBreakdown
|
|
? rubricBreakdown.weighted_score
|
|
: keywordMatchScore * 0.8 + (violations.length === 0 ? 0.2 : 0);
|
|
|
|
// --- P1-6: Negative Control logic ---
|
|
if (isNegativeControl) {
|
|
// For negative control: pass when must_contain items are ABSENT and no violations
|
|
const negativeControlPassed =
|
|
mustContainMatches.length === 0 && violations.length === 0;
|
|
|
|
return {
|
|
passed: negativeControlPassed,
|
|
keywordMatchScore,
|
|
reasoningQualityScore: negativeControlPassed ? 1.0 : 0.0,
|
|
details: {
|
|
must_contain_matches: mustContainMatches,
|
|
must_contain_misses: mustContainMisses,
|
|
must_not_contain_violations: violations,
|
|
regex_matches: regexMatches,
|
|
regex_misses: regexMisses,
|
|
severity_matched: severityMatched,
|
|
finding_count_matched: findingCountMatched,
|
|
negative_control_passed: negativeControlPassed,
|
|
finding_count_actual: findingCountActual,
|
|
schema_validation_passed: schemaValidationPassed,
|
|
rubric_breakdown: rubricBreakdown,
|
|
adaptive_keywords_extracted:
|
|
adaptiveKeywords.length > 0 ? adaptiveKeywords : undefined,
|
|
},
|
|
};
|
|
}
|
|
|
|
// --- Standard pass/fail determination ---
|
|
const rubricGate = rubricBreakdown
|
|
? rubricBreakdown.weighted_score >= threshold
|
|
: true;
|
|
|
|
const passed =
|
|
keywordMatchScore >= threshold &&
|
|
violations.length === 0 &&
|
|
findingCountMatched &&
|
|
rubricGate &&
|
|
(schemaValidationPassed === undefined || schemaValidationPassed) &&
|
|
(config?.reasoning_quality_min === undefined ||
|
|
reasoningQualityScore >= config.reasoning_quality_min) &&
|
|
(config?.allow_partial || mustContainMisses.length === 0);
|
|
|
|
return {
|
|
passed,
|
|
keywordMatchScore,
|
|
reasoningQualityScore,
|
|
details: {
|
|
must_contain_matches: mustContainMatches,
|
|
must_contain_misses: mustContainMisses,
|
|
must_not_contain_violations: violations,
|
|
regex_matches: regexMatches,
|
|
regex_misses: regexMisses,
|
|
severity_matched: severityMatched,
|
|
finding_count_matched: findingCountMatched,
|
|
finding_count_actual: findingCountActual,
|
|
schema_validation_passed: schemaValidationPassed,
|
|
rubric_breakdown: rubricBreakdown,
|
|
adaptive_keywords_extracted:
|
|
adaptiveKeywords.length > 0 ? adaptiveKeywords : undefined,
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Extract patterns from results for learning
|
|
*/
|
|
private extractPatterns(results: TestCaseResult[]): unknown[] {
|
|
return results
|
|
.filter((r) => r.passed && !r.skipped)
|
|
.map((r) => ({
|
|
testId: r.id,
|
|
category: r.category,
|
|
keywordMatchScore: r.keyword_match_score,
|
|
reasoningQualityScore: r.reasoning_quality_score,
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* Run full evaluation across all configured models
|
|
*/
|
|
async runFull(): Promise<EvalRunResult> {
|
|
const runId = `eval-${this.suite.skill}-${Date.now()}`;
|
|
const modelResults: ModelEvalResult[] = [];
|
|
|
|
console.log(`\nStarting evaluation run: ${runId}`);
|
|
console.log(`Skill: ${this.suite.skill} v${this.suite.version}`);
|
|
console.log(`Models: ${this.suite.models_to_test.join(', ')}`);
|
|
console.log(`Test cases: ${this.suite.test_cases.length}`);
|
|
|
|
// Run for each model
|
|
for (const model of this.suite.models_to_test) {
|
|
const result = await this.runForModel(model);
|
|
modelResults.push(result);
|
|
}
|
|
|
|
// Calculate cross-model variance
|
|
const passRates = modelResults.map((r) => r.pass_rate);
|
|
const maxRate = Math.max(...passRates);
|
|
const minRate = Math.min(...passRates);
|
|
const variance = maxRate - minRate;
|
|
|
|
// Share learning with fleet (MCP integration)
|
|
if (this.suite.mcp_integration?.share_learning) {
|
|
await this.mcpClient.shareLearning(
|
|
this.suite.skill,
|
|
this.suite.mcp_integration.target_agents || [],
|
|
{
|
|
passRates,
|
|
variance,
|
|
bestModel: modelResults.find((r) => r.pass_rate === maxRate)?.model,
|
|
}
|
|
);
|
|
}
|
|
|
|
// Update quality gate (MCP integration)
|
|
if (this.suite.mcp_integration?.update_quality_gate) {
|
|
const avgPassRate = passRates.reduce((a, b) => a + b, 0) / passRates.length;
|
|
await this.mcpClient.updateQualityGate(this.suite.skill, {
|
|
passRate: avgPassRate,
|
|
schemaCompliance: true,
|
|
validatorPassed: true,
|
|
evalSuiteScore: avgPassRate * 100,
|
|
});
|
|
}
|
|
|
|
// Determine overall success
|
|
const overallPassed =
|
|
modelResults.every((r) => r.success_criteria_met) &&
|
|
(!this.suite.success_criteria.cross_model_variance ||
|
|
variance <= this.suite.success_criteria.cross_model_variance);
|
|
|
|
// Generate recommendations
|
|
const recommendations: string[] = [];
|
|
if (variance > 0.1) {
|
|
recommendations.push(
|
|
`High cross-model variance (${(variance * 100).toFixed(1)}%). Consider model-specific tuning.`
|
|
);
|
|
}
|
|
const failingModels = modelResults.filter((r) => !r.success_criteria_met);
|
|
if (failingModels.length > 0) {
|
|
recommendations.push(
|
|
`Models failing criteria: ${failingModels.map((r) => r.model).join(', ')}`
|
|
);
|
|
}
|
|
|
|
return {
|
|
skill: this.suite.skill,
|
|
version: this.suite.version,
|
|
run_id: runId,
|
|
timestamp: new Date().toISOString(),
|
|
models_tested: this.suite.models_to_test,
|
|
model_results: modelResults,
|
|
cross_model_variance: variance,
|
|
overall_passed: overallPassed,
|
|
summary: {
|
|
best_model: modelResults.find((r) => r.pass_rate === maxRate)?.model || '',
|
|
worst_model: modelResults.find((r) => r.pass_rate === minRate)?.model || '',
|
|
avg_pass_rate: passRates.reduce((a, b) => a + b, 0) / passRates.length,
|
|
recommendations,
|
|
},
|
|
mcp_integration_log: this.mcpClient.getLog(),
|
|
};
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// CLI Interface
|
|
// =============================================================================
|
|
|
|
interface CLIOptions {
|
|
skill?: string;
|
|
evalFile?: string;
|
|
model?: string;
|
|
allModels?: boolean;
|
|
output?: string;
|
|
verbose?: boolean;
|
|
dryRun?: boolean;
|
|
useMcp?: boolean;
|
|
}
|
|
|
|
function parseArgs(args: string[]): CLIOptions {
|
|
const options: CLIOptions = {};
|
|
|
|
for (let i = 0; i < args.length; i++) {
|
|
const arg = args[i];
|
|
|
|
switch (arg) {
|
|
case '--skill':
|
|
case '-s':
|
|
options.skill = args[++i];
|
|
break;
|
|
case '--eval-file':
|
|
case '-e':
|
|
options.evalFile = args[++i];
|
|
break;
|
|
case '--model':
|
|
case '-m':
|
|
options.model = args[++i];
|
|
break;
|
|
case '--all-models':
|
|
case '-a':
|
|
options.allModels = true;
|
|
break;
|
|
case '--output':
|
|
case '-o':
|
|
options.output = args[++i];
|
|
break;
|
|
case '--verbose':
|
|
case '-v':
|
|
options.verbose = true;
|
|
break;
|
|
case '--dry-run':
|
|
options.dryRun = true;
|
|
break;
|
|
case '--use-mcp':
|
|
options.useMcp = true;
|
|
break;
|
|
case '--help':
|
|
case '-h':
|
|
printUsage();
|
|
process.exit(0);
|
|
}
|
|
}
|
|
|
|
return options;
|
|
}
|
|
|
|
function printUsage(): void {
|
|
console.log(`
|
|
AQE Skill Evaluation Runner
|
|
|
|
Usage:
|
|
npx tsx scripts/run-skill-eval.ts [options]
|
|
|
|
Options:
|
|
-s, --skill <name> Skill name (looks for .claude/skills/<name>/evals/<name>.yaml)
|
|
-e, --eval-file <path> Path to eval YAML file (alternative to --skill)
|
|
-m, --model <model> Run against specific model only
|
|
-a, --all-models Run against all configured models (default)
|
|
-o, --output <path> Output JSON results to file
|
|
-v, --verbose Include raw LLM output in results
|
|
--dry-run Parse and validate eval file without running
|
|
--use-mcp Enable MCP integration for shared learning
|
|
-h, --help Show this help message
|
|
|
|
Examples:
|
|
# Run security-testing evals against default model
|
|
npx tsx scripts/run-skill-eval.ts --skill security-testing
|
|
|
|
# Run against specific model
|
|
npx tsx scripts/run-skill-eval.ts --skill security-testing --model claude-sonnet-4-6
|
|
|
|
# Run with MCP integration for learning
|
|
npx tsx scripts/run-skill-eval.ts --skill security-testing --use-mcp
|
|
|
|
# Output results to file
|
|
npx tsx scripts/run-skill-eval.ts --skill security-testing --output results.json
|
|
|
|
MCP Integration:
|
|
When --use-mcp is enabled, the runner will:
|
|
1. Query existing patterns before running (mcp__agentic-qe__memory_query)
|
|
2. Track each test outcome (mcp__agentic-qe__test_outcome_track)
|
|
3. Store successful patterns after (mcp__agentic-qe__memory_store)
|
|
4. Share learning with fleet (mcp__agentic-qe__memory_share)
|
|
5. Update quality gate metrics (mcp__agentic-qe__quality_assess)
|
|
`);
|
|
}
|
|
|
|
async function loadEvalSuite(options: CLIOptions): Promise<EvalSuite> {
|
|
let evalPath: string;
|
|
|
|
if (options.evalFile) {
|
|
evalPath = options.evalFile;
|
|
} else if (options.skill) {
|
|
// Look for eval file in standard location
|
|
const possiblePaths = [
|
|
`.claude/skills/${options.skill}/evals/${options.skill}.yaml`,
|
|
`.claude/skills/${options.skill}/evals/${options.skill}.yml`,
|
|
`docs/templates/${options.skill}-eval.template.yaml`,
|
|
];
|
|
|
|
const foundPath = possiblePaths.find((p) => existsSync(p));
|
|
if (!foundPath) {
|
|
throw new Error(
|
|
`Eval file not found for skill '${options.skill}'. Searched:\n ${possiblePaths.join('\n ')}`
|
|
);
|
|
}
|
|
evalPath = foundPath;
|
|
} else {
|
|
throw new Error('Either --skill or --eval-file must be specified');
|
|
}
|
|
|
|
console.log(`Loading eval suite from: ${evalPath}`);
|
|
|
|
const content = readFileSync(evalPath, 'utf-8');
|
|
|
|
// Dynamic import of yaml parser
|
|
if (!yaml) {
|
|
yaml = await import('yaml');
|
|
}
|
|
|
|
const suite = yaml.parse(content) as EvalSuite;
|
|
|
|
// Apply CLI overrides
|
|
if (options.model && !options.allModels) {
|
|
suite.models_to_test = [options.model];
|
|
}
|
|
|
|
if (options.useMcp) {
|
|
suite.mcp_integration = {
|
|
enabled: true,
|
|
namespace: 'skill-validation',
|
|
store_patterns: true,
|
|
query_patterns: true,
|
|
track_outcomes: true,
|
|
share_learning: true,
|
|
update_quality_gate: true,
|
|
target_agents: ['qe-learning-coordinator', 'qe-queen-coordinator'],
|
|
...suite.mcp_integration,
|
|
};
|
|
}
|
|
|
|
return suite;
|
|
}
|
|
|
|
function printResults(result: EvalRunResult): void {
|
|
console.log(`\n${'='.repeat(60)}`);
|
|
console.log('EVALUATION RESULTS');
|
|
console.log(`${'='.repeat(60)}`);
|
|
|
|
console.log(`\nSkill: ${result.skill} v${result.version}`);
|
|
console.log(`Run ID: ${result.run_id}`);
|
|
console.log(`Timestamp: ${result.timestamp}`);
|
|
console.log(`Overall: ${result.overall_passed ? 'PASSED' : 'FAILED'}`);
|
|
|
|
console.log(`\nCross-Model Summary:`);
|
|
console.log(` Models tested: ${result.models_tested.join(', ')}`);
|
|
console.log(` Best model: ${result.summary.best_model}`);
|
|
console.log(` Worst model: ${result.summary.worst_model}`);
|
|
console.log(` Avg pass rate: ${(result.summary.avg_pass_rate * 100).toFixed(1)}%`);
|
|
console.log(` Cross-model variance: ${(result.cross_model_variance * 100).toFixed(1)}%`);
|
|
|
|
for (const modelResult of result.model_results) {
|
|
console.log(`\n--- ${modelResult.model} ---`);
|
|
console.log(` Pass rate: ${(modelResult.pass_rate * 100).toFixed(1)}%`);
|
|
console.log(` Critical pass rate: ${(modelResult.critical_pass_rate * 100).toFixed(1)}%`);
|
|
console.log(` Avg reasoning quality: ${modelResult.avg_reasoning_quality.toFixed(2)}`);
|
|
console.log(` Tests: ${modelResult.passed}/${modelResult.total_tests} passed, ${modelResult.skipped} skipped`);
|
|
console.log(` Execution time: ${modelResult.total_execution_time_ms}ms`);
|
|
console.log(` Criteria met: ${modelResult.success_criteria_met ? 'YES' : 'NO'}`);
|
|
|
|
if (modelResult.criteria_failures.length > 0) {
|
|
console.log(` Failures:`);
|
|
for (const failure of modelResult.criteria_failures) {
|
|
console.log(` - ${failure}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (result.summary.recommendations.length > 0) {
|
|
console.log(`\nRecommendations:`);
|
|
for (const rec of result.summary.recommendations) {
|
|
console.log(` - ${rec}`);
|
|
}
|
|
}
|
|
|
|
console.log(`\nMCP Integration Log:`);
|
|
console.log(` Patterns queried: ${result.mcp_integration_log.patterns_queried}`);
|
|
console.log(` Outcomes tracked: ${result.mcp_integration_log.outcomes_tracked}`);
|
|
console.log(` Patterns stored: ${result.mcp_integration_log.patterns_stored}`);
|
|
console.log(` Learning shared: ${result.mcp_integration_log.learning_shared}`);
|
|
console.log(` Quality gate updated: ${result.mcp_integration_log.quality_gate_updated}`);
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
const args = process.argv.slice(2);
|
|
|
|
if (args.length === 0) {
|
|
printUsage();
|
|
process.exit(1);
|
|
}
|
|
|
|
const options = parseArgs(args);
|
|
|
|
try {
|
|
const suite = await loadEvalSuite(options);
|
|
|
|
console.log(`\nLoaded eval suite: ${suite.skill} v${suite.version}`);
|
|
console.log(`Test cases: ${suite.test_cases.length}`);
|
|
console.log(`Models: ${suite.models_to_test.join(', ')}`);
|
|
|
|
if (options.dryRun) {
|
|
console.log('\n[Dry run] Eval file parsed successfully. Exiting.');
|
|
process.exit(0);
|
|
}
|
|
|
|
const runner = new SkillEvaluationRunner(suite, options.verbose);
|
|
const result = await runner.runFull();
|
|
|
|
printResults(result);
|
|
|
|
// Output to file if specified
|
|
if (options.output) {
|
|
writeFileSync(options.output, JSON.stringify(result, null, 2));
|
|
console.log(`\nResults written to: ${options.output}`);
|
|
}
|
|
|
|
// Exit with appropriate code
|
|
process.exit(result.overall_passed ? 0 : 1);
|
|
} catch (error) {
|
|
console.error('\nError:', error instanceof Error ? error.message : String(error));
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
// Run if executed directly
|
|
main().catch(console.error);
|