mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Dragan Spiridonov
d95f0d7b18
The cross-model consistency matrix (models_to_test) in 54 skill evals + both eval templates pinned retired/malformed model IDs — `claude-3.5-sonnet` (malformed: dashes, not dots; resolves to Sonnet 3.5, retired Oct 2025) and `claude-3-haiku` (retired Apr 19 2026). Both 404, so those evals could not run a meaningful cross-model check. Refresh to current pinned IDs, preserving the original "primary + fast floor" design: - claude-sonnet-4-6 # primary (high accuracy) - claude-haiku-4-5 # fast floor (minimum quality) High-stakes skills additionally get the capability ceiling claude-opus-4-8 (security-testing, security-visual-testing, n8n-security-testing, pentest-validation, chaos-engineering-resilience, qe-chaos-resilience, qe-defect-intelligence) and the security eval template. Opus is scoped to these 7 + the security template rather than all 54 to avoid a fleet-wide eval-cost blowup. Also fixes the two `--model claude-3.5-sonnet/claude-3-haiku` examples in run-skill-eval.ts and a nested cross-model test-case input in qe-learning-optimization.yaml. Verified: all eval YAML parses; zero retired/deprecated Claude IDs remain across skills/evals + templates + runner. (src/domains/constants.ts MODEL_TIERS was already current — tier 1 claude-haiku-4-5 — so it is intentionally unchanged; the legacy claude-3-haiku-20240307 entries in cost-tracker/bedrock are retained historical pricing/availability data.) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
367 lines
11 KiB
YAML
367 lines
11 KiB
YAML
# =============================================================================
|
|
# AQE Skill Evaluation Test Suite Template
|
|
# Copy this template to: .claude/skills/{skill-name}/evals/{skill-name}.yaml
|
|
# =============================================================================
|
|
#
|
|
# This evaluation suite validates skill behavior through:
|
|
# 1. Input/expected-output test cases
|
|
# 2. Multi-model consistency testing
|
|
# 3. Semantic validation of reasoning quality
|
|
# 4. AQE MCP integration for shared learning (NEW)
|
|
# 5. ReasoningBank pattern storage (NEW)
|
|
#
|
|
# Schema: docs/schemas/skill-eval.schema.json
|
|
# MCP Spec: docs/specs/skill-validation-mcp-integration.md
|
|
# Runner: scripts/run-skill-eval.ts
|
|
#
|
|
# For a comprehensive example, see: docs/templates/security-testing-eval.template.yaml
|
|
# =============================================================================
|
|
|
|
skill: REPLACE_WITH_SKILL_NAME
|
|
version: 1.0.0
|
|
description: >
|
|
Evaluation test suite for REPLACE_WITH_SKILL_NAME skill.
|
|
Tests core functionality across multiple models to ensure consistent,
|
|
high-quality output.
|
|
|
|
# =============================================================================
|
|
# Multi-Model Configuration
|
|
# =============================================================================
|
|
# Test across multiple models to ensure consistent behavior and identify
|
|
# model-specific quirks. Results are compared to detect variance.
|
|
|
|
models_to_test:
|
|
- claude-sonnet-4-6 # Primary (high accuracy expected)
|
|
- claude-haiku-4-5 # Fast model (minimum quality floor)
|
|
# - gpt-4o # Optional: cross-vendor validation
|
|
|
|
# =============================================================================
|
|
# MCP Integration Configuration (NEW in v1.4)
|
|
# =============================================================================
|
|
# Per docs/specs/skill-validation-mcp-integration.md
|
|
# Enable to integrate with AQE ReasoningBank for shared learning.
|
|
|
|
mcp_integration:
|
|
enabled: true
|
|
namespace: skill-validation
|
|
|
|
# Query existing patterns before running evals
|
|
query_patterns: true
|
|
|
|
# Track each test outcome for the learning feedback loop
|
|
track_outcomes: true
|
|
|
|
# Store successful patterns after evals complete
|
|
store_patterns: true
|
|
|
|
# Share learning with fleet coordinator agents
|
|
share_learning: true
|
|
|
|
# Update quality gate with validation metrics
|
|
update_quality_gate: true
|
|
|
|
# Agents to share learning with
|
|
target_agents:
|
|
- qe-learning-coordinator
|
|
- qe-queen-coordinator
|
|
|
|
# =============================================================================
|
|
# ReasoningBank Learning Configuration (NEW in v1.4)
|
|
# =============================================================================
|
|
|
|
learning:
|
|
store_success_patterns: true
|
|
store_failure_patterns: true
|
|
pattern_ttl_days: 90
|
|
min_confidence_to_store: 0.7
|
|
cross_model_comparison: true
|
|
|
|
# =============================================================================
|
|
# Result Format Configuration (NEW in v1.4)
|
|
# =============================================================================
|
|
|
|
result_format:
|
|
json_output: true
|
|
markdown_report: false
|
|
include_raw_output: false
|
|
include_timing: true
|
|
include_token_usage: true
|
|
|
|
# =============================================================================
|
|
# Environment Setup
|
|
# =============================================================================
|
|
|
|
setup:
|
|
required_tools: []
|
|
# - tool_name
|
|
environment_variables: {}
|
|
# ENV_VAR: value
|
|
fixtures: []
|
|
# - name: fixture_name
|
|
# path: fixtures/fixture_name.json
|
|
# content: |
|
|
# { "key": "value" }
|
|
|
|
# =============================================================================
|
|
# Test Cases
|
|
# =============================================================================
|
|
# Each test case validates a specific behavior or scenario.
|
|
# IDs follow the pattern: tc{NNN}_{short_description}
|
|
# =============================================================================
|
|
|
|
test_cases:
|
|
# -------------------------------------------------------------------------
|
|
# Basic Functionality Tests
|
|
# -------------------------------------------------------------------------
|
|
|
|
- id: tc001_basic_invocation
|
|
description: "Skill responds to basic invocation with valid output"
|
|
category: basic
|
|
priority: critical
|
|
|
|
input:
|
|
prompt: |
|
|
Analyze the following code for issues:
|
|
```javascript
|
|
function hello() {
|
|
console.log("Hello, World!");
|
|
}
|
|
```
|
|
context:
|
|
language: javascript
|
|
|
|
expected_output:
|
|
must_contain:
|
|
- "function"
|
|
- "hello"
|
|
must_not_contain:
|
|
- "error"
|
|
- "unable to analyze"
|
|
# finding_count:
|
|
# min: 0
|
|
# max: 5
|
|
|
|
validation:
|
|
schema_check: true
|
|
keyword_match_threshold: 0.8
|
|
reasoning_quality_min: 0.6
|
|
|
|
- id: tc002_handles_empty_input
|
|
description: "Skill handles empty or minimal input gracefully"
|
|
category: edge_cases
|
|
priority: high
|
|
|
|
input:
|
|
prompt: "Analyze this code:"
|
|
context:
|
|
language: unknown
|
|
|
|
expected_output:
|
|
must_contain:
|
|
- "no code"
|
|
- "provide"
|
|
must_not_contain:
|
|
- "exception"
|
|
- "crash"
|
|
|
|
validation:
|
|
schema_check: true
|
|
allow_partial: true
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Core Functionality Tests - CUSTOMIZE THESE
|
|
# -------------------------------------------------------------------------
|
|
|
|
- id: tc003_core_feature_1
|
|
description: "DESCRIBE WHAT THIS TEST VALIDATES"
|
|
category: core
|
|
priority: high
|
|
|
|
input:
|
|
code: |
|
|
// Replace with relevant code sample
|
|
const example = "test";
|
|
context:
|
|
language: javascript
|
|
framework: nodejs
|
|
|
|
expected_output:
|
|
must_contain:
|
|
- "EXPECTED_KEYWORD_1"
|
|
- "EXPECTED_KEYWORD_2"
|
|
severity_classification: medium
|
|
|
|
validation:
|
|
schema_check: true
|
|
keyword_match_threshold: 0.8
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Negative Tests (Should NOT find issues)
|
|
# -------------------------------------------------------------------------
|
|
|
|
- id: tc010_no_false_positives
|
|
description: "Skill does not flag clean code as problematic"
|
|
category: negative
|
|
priority: high
|
|
|
|
input:
|
|
code: |
|
|
// Well-written, clean code
|
|
function add(a, b) {
|
|
return a + b;
|
|
}
|
|
context:
|
|
language: javascript
|
|
|
|
expected_output:
|
|
must_contain:
|
|
- "no issues"
|
|
# OR
|
|
# - "clean"
|
|
# - "good"
|
|
must_not_contain:
|
|
- "critical"
|
|
- "vulnerability"
|
|
- "error"
|
|
|
|
validation:
|
|
schema_check: true
|
|
finding_count:
|
|
max: 1 # Allow at most 1 minor finding
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Edge Cases
|
|
# -------------------------------------------------------------------------
|
|
|
|
- id: tc020_large_input
|
|
description: "Skill handles large input without truncation issues"
|
|
category: edge_cases
|
|
priority: medium
|
|
skip: false
|
|
|
|
input:
|
|
file_path: fixtures/large_sample.js
|
|
context:
|
|
language: javascript
|
|
|
|
expected_output:
|
|
must_contain:
|
|
- "analyzed"
|
|
|
|
validation:
|
|
schema_check: true
|
|
|
|
timeout_ms: 60000 # Longer timeout for large files
|
|
|
|
- id: tc021_special_characters
|
|
description: "Skill handles special characters in input"
|
|
category: edge_cases
|
|
priority: medium
|
|
|
|
input:
|
|
code: |
|
|
const emoji = "Hello 🌍!";
|
|
const unicode = "日本語テスト";
|
|
const escape = "Line1\nLine2\tTabbed";
|
|
context:
|
|
language: javascript
|
|
|
|
expected_output:
|
|
must_not_contain:
|
|
- "encoding error"
|
|
- "parse error"
|
|
|
|
validation:
|
|
schema_check: true
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Multi-Language Support (if applicable)
|
|
# -------------------------------------------------------------------------
|
|
|
|
- id: tc030_python_support
|
|
description: "Skill correctly analyzes Python code"
|
|
category: language_support
|
|
priority: medium
|
|
skip: true # Enable if skill supports Python
|
|
|
|
input:
|
|
code: |
|
|
def hello():
|
|
print("Hello, World!")
|
|
context:
|
|
language: python
|
|
|
|
expected_output:
|
|
must_contain:
|
|
- "def"
|
|
- "function"
|
|
|
|
validation:
|
|
schema_check: true
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Integration Scenarios
|
|
# -------------------------------------------------------------------------
|
|
|
|
- id: tc040_with_context
|
|
description: "Skill uses provided context appropriately"
|
|
category: integration
|
|
priority: medium
|
|
|
|
input:
|
|
code: |
|
|
app.get('/api/users', (req, res) => {
|
|
const users = db.query('SELECT * FROM users');
|
|
res.json(users);
|
|
});
|
|
context:
|
|
language: javascript
|
|
framework: express
|
|
environment: production
|
|
options:
|
|
detailed: true
|
|
|
|
expected_output:
|
|
must_contain:
|
|
- "express"
|
|
- "api"
|
|
|
|
validation:
|
|
schema_check: true
|
|
grading_rubric:
|
|
completeness: 0.4
|
|
accuracy: 0.4
|
|
actionability: 0.2
|
|
|
|
# =============================================================================
|
|
# Success Criteria
|
|
# =============================================================================
|
|
# Define what constitutes a passing evaluation suite run.
|
|
# =============================================================================
|
|
|
|
success_criteria:
|
|
# Minimum percentage of tests that must pass
|
|
pass_rate: 0.9
|
|
|
|
# Critical tests must have 100% pass rate
|
|
critical_pass_rate: 1.0
|
|
|
|
# Average reasoning quality across all tests
|
|
avg_reasoning_quality: 0.7
|
|
|
|
# Maximum time for entire suite (5 minutes)
|
|
max_execution_time_ms: 300000
|
|
|
|
# Maximum variance between different models (0.1 = 10%)
|
|
cross_model_variance: 0.15
|
|
|
|
# =============================================================================
|
|
# Metadata
|
|
# =============================================================================
|
|
|
|
metadata:
|
|
author: "@your-github-handle"
|
|
created: "2026-02-02"
|
|
last_updated: "2026-02-02"
|
|
coverage_target: "Core functionality and common edge cases"
|