mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Dragan Spiridonov
af4140efb9
Apply Phoenix Architecture principles to AQE — regenerate internals, conserve the interface. ADR-113 (evals are oracles, durable-first tests, mutation-score gate): - src/validation/oracle-eval.ts grades a generated test by running it against a reference impl + operator mutants (reuses qe-arena, ADR-104); rejects assertion-less output - oracle mode wired into scripts/run-skill-eval.ts (skips under the simulating runner; runs live) - qe-test-generation rewritten durable-first; its eval flipped to oracle - src/feedback/regenerability-gate.ts: mutation-score + regenerability gate (warn-default, opt-in block) - live-validated: Claude sonnet-4-6 clears the oracle 5/5; weak local-model output correctly fails ADR-114 (conservation layer, Phoenix essay 14): - src/validation/conservation-guard.ts + scripts/conservation-guard.ts guard CLI commands, output schemas, and dashboard API against breaking removals (deprecation registry exempts) - policy doc + baselines in verification/conservation/; wired into invariant-check.yml non-blocking LLM provider fixes: - OpenRouterProvider.type was 'openai' but registered under 'openrouter' -> ProviderManager failover never found it; fixed to 'openrouter' end-to-end - Gemini default model gemini-1.5-pro was retired -> gemini-2.5-flash; models/pricing refreshed from the live Gemini ListModels API - scripts/provider-health.ts (npm run providers:health) Cheaper-model eval lanes: - scripts/oracle-model-bench.ts + openrouter-models.ts; openai/gpt-oss-120b clears the oracle ~50-70x cheaper than Claude Sonnet; guide in docs/guides/cheaper-model-eval-lanes.md Verification: npm run build ok; 115 new+touched tests pass; 724 LLM tests pass; lint + parity clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
597 lines
17 KiB
JSON
597 lines
17 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://agentic-qe.dev/schemas/skill-eval.json",
|
|
"title": "AQE Skill Evaluation Test Suite Schema",
|
|
"description": "Schema for skill evaluation YAML test suites used in Trust Tier 3 validation. Supports multi-model testing, AQE MCP integration, and ReasoningBank learning.",
|
|
"type": "object",
|
|
"required": [
|
|
"skill",
|
|
"version",
|
|
"test_cases",
|
|
"success_criteria"
|
|
],
|
|
"properties": {
|
|
"skill": {
|
|
"type": "string",
|
|
"pattern": "^[a-z][a-z0-9-]*$",
|
|
"description": "Skill name being evaluated (must match .claude/skills/{skill}/SKILL.md)"
|
|
},
|
|
"version": {
|
|
"type": "string",
|
|
"pattern": "^\\d+\\.\\d+\\.\\d+$",
|
|
"description": "Evaluation suite version (semver)"
|
|
},
|
|
"description": {
|
|
"type": "string",
|
|
"maxLength": 1000,
|
|
"description": "Description of evaluation suite purpose and coverage goals"
|
|
},
|
|
"models_to_test": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"enum": [
|
|
"claude-opus-4-8",
|
|
"claude-opus-4-7",
|
|
"claude-sonnet-4-6",
|
|
"claude-haiku-4-5",
|
|
"claude-opus-4-5",
|
|
"gpt-4o",
|
|
"gpt-4o-mini",
|
|
"gpt-4-turbo"
|
|
]
|
|
},
|
|
"default": [
|
|
"claude-sonnet-4-6"
|
|
],
|
|
"minItems": 1,
|
|
"uniqueItems": true,
|
|
"description": "Models to run evaluation against for cross-model validation"
|
|
},
|
|
"mcp_integration": {
|
|
"$ref": "#/$defs/mcp_integration_config",
|
|
"description": "AQE MCP integration configuration for shared learning"
|
|
},
|
|
"learning": {
|
|
"$ref": "#/$defs/learning_config",
|
|
"description": "ReasoningBank learning configuration"
|
|
},
|
|
"result_format": {
|
|
"$ref": "#/$defs/result_format",
|
|
"description": "Output format configuration"
|
|
},
|
|
"setup": {
|
|
"type": "object",
|
|
"description": "Setup configuration for test environment",
|
|
"properties": {
|
|
"required_tools": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Tools that must be available"
|
|
},
|
|
"environment_variables": {
|
|
"type": "object",
|
|
"additionalProperties": {
|
|
"type": "string"
|
|
},
|
|
"description": "Environment variables to set"
|
|
},
|
|
"fixtures": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"name": {
|
|
"type": "string"
|
|
},
|
|
"path": {
|
|
"type": "string"
|
|
},
|
|
"content": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
},
|
|
"description": "Test fixtures to create"
|
|
}
|
|
}
|
|
},
|
|
"test_cases": {
|
|
"type": "array",
|
|
"minItems": 1,
|
|
"items": {
|
|
"$ref": "#/$defs/test_case"
|
|
},
|
|
"description": "Evaluation test cases"
|
|
},
|
|
"success_criteria": {
|
|
"type": "object",
|
|
"required": [
|
|
"pass_rate"
|
|
],
|
|
"properties": {
|
|
"pass_rate": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"description": "Minimum pass rate for suite to pass"
|
|
},
|
|
"critical_pass_rate": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"default": 1.0,
|
|
"description": "Pass rate required for critical priority tests"
|
|
},
|
|
"avg_reasoning_quality": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"description": "Minimum average reasoning quality score"
|
|
},
|
|
"max_execution_time_ms": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Maximum execution time for test suite"
|
|
},
|
|
"cross_model_variance": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"description": "Maximum allowed variance between models"
|
|
}
|
|
}
|
|
},
|
|
"metadata": {
|
|
"type": "object",
|
|
"properties": {
|
|
"author": {
|
|
"type": "string"
|
|
},
|
|
"created": {
|
|
"type": "string",
|
|
"format": "date"
|
|
},
|
|
"last_updated": {
|
|
"type": "string",
|
|
"format": "date"
|
|
},
|
|
"coverage_target": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"$defs": {
|
|
"test_case": {
|
|
"type": "object",
|
|
"required": [
|
|
"id",
|
|
"description",
|
|
"input",
|
|
"expected_output"
|
|
],
|
|
"properties": {
|
|
"id": {
|
|
"type": "string",
|
|
"pattern": "^tc\\d{3}_[a-z_]+$",
|
|
"description": "Test case ID (e.g., tc001_sql_injection)"
|
|
},
|
|
"description": {
|
|
"type": "string",
|
|
"description": "What this test case validates"
|
|
},
|
|
"category": {
|
|
"type": "string",
|
|
"description": "Test category for grouping"
|
|
},
|
|
"priority": {
|
|
"type": "string",
|
|
"enum": [
|
|
"critical",
|
|
"high",
|
|
"medium",
|
|
"low"
|
|
],
|
|
"default": "medium",
|
|
"description": "Test priority"
|
|
},
|
|
"skip": {
|
|
"type": "boolean",
|
|
"default": false,
|
|
"description": "Skip this test case"
|
|
},
|
|
"skip_reason": {
|
|
"type": "string",
|
|
"description": "Reason for skipping"
|
|
},
|
|
"input": {
|
|
"$ref": "#/$defs/test_input"
|
|
},
|
|
"expected_output": {
|
|
"$ref": "#/$defs/expected_output"
|
|
},
|
|
"validation": {
|
|
"$ref": "#/$defs/validation_config"
|
|
},
|
|
"timeout_ms": {
|
|
"type": "integer",
|
|
"minimum": 1000,
|
|
"default": 30000,
|
|
"description": "Test timeout in milliseconds"
|
|
},
|
|
"oracle": {
|
|
"$ref": "#/$defs/oracle_config"
|
|
}
|
|
}
|
|
},
|
|
"test_input": {
|
|
"type": "object",
|
|
"description": "Input provided to the skill",
|
|
"properties": {
|
|
"code": {
|
|
"type": "string",
|
|
"description": "Code snippet to analyze"
|
|
},
|
|
"file_path": {
|
|
"type": "string",
|
|
"description": "Path to file to analyze"
|
|
},
|
|
"url": {
|
|
"type": "string",
|
|
"format": "uri",
|
|
"description": "URL to analyze"
|
|
},
|
|
"prompt": {
|
|
"type": "string",
|
|
"description": "Custom prompt for the skill"
|
|
},
|
|
"context": {
|
|
"type": "object",
|
|
"properties": {
|
|
"language": {
|
|
"type": "string",
|
|
"description": "Programming language"
|
|
},
|
|
"framework": {
|
|
"type": "string",
|
|
"description": "Framework in use"
|
|
},
|
|
"environment": {
|
|
"type": "string",
|
|
"enum": [
|
|
"development",
|
|
"staging",
|
|
"production"
|
|
],
|
|
"description": "Target environment"
|
|
}
|
|
}
|
|
},
|
|
"options": {
|
|
"type": "object",
|
|
"additionalProperties": true,
|
|
"description": "Skill-specific options"
|
|
}
|
|
}
|
|
},
|
|
"expected_output": {
|
|
"type": "object",
|
|
"description": "Expected characteristics of skill output",
|
|
"properties": {
|
|
"must_contain": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Strings that must appear in output"
|
|
},
|
|
"must_not_contain": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Strings that must not appear in output"
|
|
},
|
|
"must_match_regex": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "Regex patterns output must match"
|
|
},
|
|
"severity_classification": {
|
|
"type": "string",
|
|
"enum": [
|
|
"critical",
|
|
"high",
|
|
"medium",
|
|
"low",
|
|
"info"
|
|
],
|
|
"description": "Expected severity classification"
|
|
},
|
|
"finding_count": {
|
|
"type": "object",
|
|
"properties": {
|
|
"min": {
|
|
"type": "integer",
|
|
"minimum": 0
|
|
},
|
|
"max": {
|
|
"type": "integer",
|
|
"minimum": 0
|
|
}
|
|
},
|
|
"description": "Expected range of findings"
|
|
},
|
|
"recommendation_count": {
|
|
"type": "object",
|
|
"properties": {
|
|
"min": {
|
|
"type": "integer",
|
|
"minimum": 0
|
|
},
|
|
"max": {
|
|
"type": "integer",
|
|
"minimum": 0
|
|
}
|
|
},
|
|
"description": "Expected range of recommendations"
|
|
},
|
|
"schema_path": {
|
|
"type": "string",
|
|
"description": "Path to JSON schema for output validation"
|
|
},
|
|
"custom_assertions": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"type": {
|
|
"type": "string",
|
|
"enum": [
|
|
"jsonpath",
|
|
"semantic",
|
|
"function"
|
|
]
|
|
},
|
|
"expression": {
|
|
"type": "string"
|
|
},
|
|
"expected": {}
|
|
}
|
|
},
|
|
"description": "Custom assertions for complex validation"
|
|
}
|
|
}
|
|
},
|
|
"validation_config": {
|
|
"type": "object",
|
|
"description": "Validation configuration for test case",
|
|
"properties": {
|
|
"schema_check": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Validate output against schema"
|
|
},
|
|
"keyword_match_threshold": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"default": 0.8,
|
|
"description": "Minimum keyword match ratio for pass"
|
|
},
|
|
"reasoning_quality_min": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"description": "Minimum reasoning quality score"
|
|
},
|
|
"semantic_similarity_min": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"description": "Minimum semantic similarity to expected"
|
|
},
|
|
"allow_partial": {
|
|
"type": "boolean",
|
|
"default": false,
|
|
"description": "Allow partial matches"
|
|
},
|
|
"grading_rubric": {
|
|
"type": "object",
|
|
"properties": {
|
|
"completeness": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1
|
|
},
|
|
"accuracy": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1
|
|
},
|
|
"actionability": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1
|
|
}
|
|
},
|
|
"description": "Weighted grading rubric (weights should sum to 1.0)"
|
|
},
|
|
"oracle": {
|
|
"type": "boolean",
|
|
"default": false,
|
|
"description": "Grade this case as an oracle (run+mutate) instead of keyword/semantic matching. Requires test_case.oracle."
|
|
},
|
|
"mutation_score_min": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"description": "Minimum mutation kill rate for an oracle case (overrides oracle.threshold when set)"
|
|
}
|
|
}
|
|
},
|
|
"mcp_integration_config": {
|
|
"type": "object",
|
|
"description": "Configuration for AQE MCP tool integration per MCP Integration Spec",
|
|
"properties": {
|
|
"enabled": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Enable MCP integration for this eval suite"
|
|
},
|
|
"namespace": {
|
|
"type": "string",
|
|
"default": "skill-validation",
|
|
"description": "Memory namespace for storing patterns and outcomes"
|
|
},
|
|
"store_patterns": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Store successful patterns via mcp__agentic-qe__memory_store"
|
|
},
|
|
"query_patterns": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Query existing patterns before running via mcp__agentic-qe__memory_query"
|
|
},
|
|
"track_outcomes": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Track test outcomes via mcp__agentic-qe__test_outcome_track"
|
|
},
|
|
"share_learning": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Share learning with fleet via mcp__agentic-qe__memory_share"
|
|
},
|
|
"update_quality_gate": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Update quality gate metrics via mcp__agentic-qe__quality_assess"
|
|
},
|
|
"target_agents": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"default": [
|
|
"qe-learning-coordinator",
|
|
"qe-queen-coordinator"
|
|
],
|
|
"description": "Agent IDs to share learning with"
|
|
}
|
|
}
|
|
},
|
|
"learning_config": {
|
|
"type": "object",
|
|
"description": "Configuration for ReasoningBank learning integration",
|
|
"properties": {
|
|
"store_success_patterns": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Store patterns from successful test runs"
|
|
},
|
|
"store_failure_patterns": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Store patterns from failures for learning"
|
|
},
|
|
"pattern_ttl_days": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"maximum": 365,
|
|
"default": 90,
|
|
"description": "Time-to-live for stored patterns in days"
|
|
},
|
|
"min_confidence_to_store": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"default": 0.7,
|
|
"description": "Minimum confidence score to store pattern"
|
|
},
|
|
"cross_model_comparison": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Store cross-model behavior comparisons"
|
|
}
|
|
}
|
|
},
|
|
"result_format": {
|
|
"type": "object",
|
|
"description": "Output format configuration for evaluation results",
|
|
"properties": {
|
|
"json_output": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Output results as JSON"
|
|
},
|
|
"markdown_report": {
|
|
"type": "boolean",
|
|
"default": false,
|
|
"description": "Generate markdown report"
|
|
},
|
|
"include_raw_output": {
|
|
"type": "boolean",
|
|
"default": false,
|
|
"description": "Include raw LLM output in results"
|
|
},
|
|
"include_timing": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Include timing information"
|
|
},
|
|
"include_token_usage": {
|
|
"type": "boolean",
|
|
"default": true,
|
|
"description": "Include token usage statistics"
|
|
}
|
|
}
|
|
},
|
|
"oracle_config": {
|
|
"type": "object",
|
|
"description": "Oracle-mode evaluation: run the skill's generated test against a reference implementation (must pass) and against operator mutants of it (each surviving mutant is a missed bug). Mutation kill rate is the pass signal. Reuses the qe-arena mutator + node --test runner (ADR-104).",
|
|
"required": [
|
|
"module_name",
|
|
"reference_impl"
|
|
],
|
|
"properties": {
|
|
"module_name": {
|
|
"type": "string",
|
|
"description": "Base name (no extension) of the module under test; the generated test imports it as ../src/<module_name>.mjs"
|
|
},
|
|
"reference_impl": {
|
|
"type": "string",
|
|
"description": "ESM source of the correct implementation the generated test is graded against"
|
|
},
|
|
"max_mutants": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "Cap on mutants evaluated (deterministic position-sorted prefix). Omit to evaluate all."
|
|
},
|
|
"threshold": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"default": 0.5,
|
|
"description": "Minimum mutation kill rate required to pass"
|
|
},
|
|
"extract": {
|
|
"type": "string",
|
|
"enum": [
|
|
"raw",
|
|
"first_code_block"
|
|
],
|
|
"default": "first_code_block",
|
|
"description": "How to obtain the test source from model output: use it verbatim, or take the first fenced code block"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|