mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Dragan Spiridonov
4a641e7ef1
Fifth post-audit sweep. 24 files, +66/-60.
CRITICAL (user-facing):
* .github/workflows/skill-validation.yml:422: CI eval runner
--model claude-sonnet-4 → claude-sonnet-4-6. Previously CI would
target a retiring model.
* src/coordination/consensus/providers/claude-provider.ts: Claude 3.x
consensus provider modernized — type union extended with
claude-sonnet-4-6 / claude-opus-4-7 / claude-haiku-4-5 as first
entries, default model updated from claude-3-5-sonnet-20241022
to claude-sonnet-4-6, availableModels list prepends new models.
Legacy 3.x entries retained for backward compat.
* src/mcp/security/sampling-server.ts: default model
claude-3-sonnet-20240229 (retired 2026-02-19) → claude-sonnet-4-6.
Skill evals (12 files, 6 + 6 mirrors):
* .claude/skills/ and assets/skills/ mirrors for:
performance-testing, chaos-engineering-resilience, quality-metrics,
localization-testing, accessibility-testing,
n8n-trigger-testing-strategies. models_to_test list updated to
claude-sonnet-4-6 / claude-haiku-4-5.
Test files (9):
* tests/unit/coordination/consensus/providers/claude-provider.test.ts:
default model, test descriptions, and references updated to 4.6/4.7.
* tests/unit/routing/economic-routing.test.ts: claude-3-opus-20240229
usage → claude-opus-4-7.
* tests/unit/learning/skill-validation-learner.test.ts: claude-3-haiku
/ claude-3.5-sonnet / claude-3-opus → new canonicals.
* tests/unit/mcp/handlers/handler-test-utils.ts: claude-3-haiku →
claude-haiku-4-5.
* tests/integration/llm/{real-provider-integration,multi-provider}.test.ts:
bare 'claude-sonnet-4' canonical → 'claude-sonnet-4-6'.
* tests/integration/validation/parallel-eval-runner.test.ts:
claude-3.5-sonnet / claude-3-haiku → new canonicals.
* tests/routing/advisor/multi-model-executor.test.ts:
CRITICAL BUG — 'claude-opus-4-6' (non-existent model ID, typo) ×2
→ 'claude-opus-4-7'. Stale test name 'anthropic/claude-opus-4
when no provider' → '-4.7'.
* tests/unit/opencode/degradation.test.ts: CRITICAL BUG —
modelId 'claude-opus-4-6' (typo) → 'claude-opus-4-7'.
Left intentionally untouched:
* tests/unit/coordination/consensus/providers/openrouter-provider.test.ts:
tests that OpenRouter provider lists claude-3-haiku / claude-3.5-sonnet
/ claude-3-opus as supported. These assertions reflect actual
OpenRouter provider data, which hasn't changed. Adding new-model
assertions would be an enhancement, not a fix.
Verification:
* 0 tsc errors
* 174/174 tests across touched paths pass
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
492 lines
16 KiB
YAML
492 lines
16 KiB
YAML
# =============================================================================
|
|
# Skill Validation CI Workflow
|
|
# =============================================================================
|
|
#
|
|
# Validates skill output schemas, validator scripts, and evaluation suites
|
|
# per ADR-056: Skill Validation Stack Architecture.
|
|
#
|
|
# Trust Tier-Based Validation:
|
|
# - Tier 3 (Verified): All validations required, blocks PR on failure
|
|
# - Tier 2 (Validated): Schema + Validator required, warns on failure
|
|
# - Tier 1 (Structured): Schema required, warns on failure
|
|
# - Tier 0 (Advisory): Skipped entirely
|
|
#
|
|
# Triggers:
|
|
# - On pull request when any skill files change
|
|
# - On manual dispatch with specific skill or "all"
|
|
#
|
|
# =============================================================================
|
|
|
|
name: Skill Validation
|
|
|
|
on:
|
|
pull_request:
|
|
paths:
|
|
- '.claude/skills/**'
|
|
branches:
|
|
- main
|
|
- develop
|
|
workflow_dispatch:
|
|
inputs:
|
|
skill:
|
|
description: 'Skill to validate (skill name or "all")'
|
|
required: true
|
|
default: 'all'
|
|
type: string
|
|
run_evals:
|
|
description: 'Run evaluation suites (requires ANTHROPIC_API_KEY)'
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
|
|
permissions:
|
|
contents: read
|
|
pull-requests: write
|
|
checks: write
|
|
|
|
concurrency:
|
|
group: skill-validation-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
NODE_VERSION: '24'
|
|
|
|
jobs:
|
|
# ===========================================================================
|
|
# Job 1: Detect Changed Skills and Their Trust Tiers
|
|
# ===========================================================================
|
|
detect-changes:
|
|
name: Detect Changed Skills
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
outputs:
|
|
skills_json: ${{ steps.detect.outputs.skills_json }}
|
|
tier3_skills: ${{ steps.detect.outputs.tier3_skills }}
|
|
tier2_skills: ${{ steps.detect.outputs.tier2_skills }}
|
|
tier1_skills: ${{ steps.detect.outputs.tier1_skills }}
|
|
has_changes: ${{ steps.detect.outputs.has_changes }}
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Detect skills and trust tiers
|
|
id: detect
|
|
run: |
|
|
echo "Detecting skills and their trust tiers..."
|
|
|
|
# Function to get trust tier from SKILL.md frontmatter
|
|
get_trust_tier() {
|
|
local skill_dir="$1"
|
|
local skill_md="$skill_dir/SKILL.md"
|
|
|
|
if [ ! -f "$skill_md" ]; then
|
|
skill_md="$skill_dir/skill.md"
|
|
fi
|
|
|
|
if [ ! -f "$skill_md" ]; then
|
|
echo "0"
|
|
return
|
|
fi
|
|
|
|
# Extract trust_tier from YAML frontmatter
|
|
tier=$(sed -n '/^---$/,/^---$/p' "$skill_md" | grep -E '^trust_tier:' | head -1 | sed 's/trust_tier:[[:space:]]*//' | tr -d ' ')
|
|
|
|
if [ -z "$tier" ]; then
|
|
echo "0"
|
|
else
|
|
echo "$tier"
|
|
fi
|
|
}
|
|
|
|
# Get list of skills to check
|
|
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
|
|
if [ "${{ inputs.skill }}" == "all" ]; then
|
|
skill_dirs=$(find .claude/skills -maxdepth 1 -mindepth 1 -type d ! -name '.*' | sort)
|
|
else
|
|
skill_dirs=".claude/skills/${{ inputs.skill }}"
|
|
fi
|
|
else
|
|
# PR trigger: detect changed skills
|
|
base_sha="${{ github.event.pull_request.base.sha }}"
|
|
head_sha="${{ github.event.pull_request.head.sha }}"
|
|
|
|
changed_files=$(git diff --name-only "$base_sha" "$head_sha" \
|
|
| grep '^.claude/skills/' \
|
|
| grep -v '^.claude/skills/\.' \
|
|
|| true)
|
|
|
|
if [ -z "$changed_files" ]; then
|
|
echo "No skill changes detected"
|
|
echo "skills_json=[]" >> $GITHUB_OUTPUT
|
|
echo "tier3_skills=[]" >> $GITHUB_OUTPUT
|
|
echo "tier2_skills=[]" >> $GITHUB_OUTPUT
|
|
echo "tier1_skills=[]" >> $GITHUB_OUTPUT
|
|
echo "has_changes=false" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
# Get unique skill directories
|
|
skill_dirs=$(echo "$changed_files" | cut -d'/' -f1-3 | sort -u | grep -v '^$')
|
|
fi
|
|
|
|
# Build arrays by tier
|
|
tier3_list=""
|
|
tier2_list=""
|
|
tier1_list=""
|
|
all_skills=""
|
|
|
|
for skill_dir in $skill_dirs; do
|
|
if [ ! -d "$skill_dir" ]; then
|
|
continue
|
|
fi
|
|
|
|
skill_name=$(basename "$skill_dir")
|
|
|
|
# Skip hidden directories and special files
|
|
if [[ "$skill_name" == .* ]] || [[ "$skill_name" == "TRUST-TIERS.md" ]] || [[ "$skill_name" == *.json ]]; then
|
|
continue
|
|
fi
|
|
|
|
tier=$(get_trust_tier "$skill_dir")
|
|
|
|
echo "Skill: $skill_name -> Tier $tier"
|
|
|
|
all_skills="$all_skills\"$skill_name\","
|
|
|
|
case "$tier" in
|
|
3) tier3_list="$tier3_list\"$skill_name\"," ;;
|
|
2) tier2_list="$tier2_list\"$skill_name\"," ;;
|
|
1) tier1_list="$tier1_list\"$skill_name\"," ;;
|
|
*) echo " (Tier 0 - skipping validation)" ;;
|
|
esac
|
|
done
|
|
|
|
# Format as JSON arrays (remove trailing comma)
|
|
all_skills="[${all_skills%,}]"
|
|
tier3_list="[${tier3_list%,}]"
|
|
tier2_list="[${tier2_list%,}]"
|
|
tier1_list="[${tier1_list%,}]"
|
|
|
|
echo "All skills: $all_skills"
|
|
echo "Tier 3 (blocking): $tier3_list"
|
|
echo "Tier 2 (warn): $tier2_list"
|
|
echo "Tier 1 (warn): $tier1_list"
|
|
|
|
echo "skills_json=$all_skills" >> $GITHUB_OUTPUT
|
|
echo "tier3_skills=$tier3_list" >> $GITHUB_OUTPUT
|
|
echo "tier2_skills=$tier2_list" >> $GITHUB_OUTPUT
|
|
echo "tier1_skills=$tier1_list" >> $GITHUB_OUTPUT
|
|
|
|
if [ "$all_skills" != "[]" ]; then
|
|
echo "has_changes=true" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "has_changes=false" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
# ===========================================================================
|
|
# Job 2: Validate Tier 3 Skills (BLOCKING)
|
|
# ===========================================================================
|
|
validate-tier3:
|
|
name: Tier 3 Validation
|
|
needs: detect-changes
|
|
if: needs.detect-changes.outputs.tier3_skills != '[]' && needs.detect-changes.outputs.tier3_skills != ''
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
strategy:
|
|
matrix:
|
|
skill: ${{ fromJson(needs.detect-changes.outputs.tier3_skills) }}
|
|
fail-fast: false
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: ${{ env.NODE_VERSION }}
|
|
|
|
- name: Install tools
|
|
run: |
|
|
npm install -g ajv-cli ajv-formats
|
|
sudo apt-get update && sudo apt-get install -y jq bc
|
|
|
|
- name: Validate Schema
|
|
id: schema
|
|
run: |
|
|
SKILL="${{ matrix.skill }}"
|
|
SCHEMA_PATH=".claude/skills/$SKILL/schemas/output.json"
|
|
|
|
echo "=== Tier 3 Schema Validation: $SKILL ==="
|
|
|
|
if [ ! -f "$SCHEMA_PATH" ]; then
|
|
echo "::error::Tier 3 skill '$SKILL' missing required schema"
|
|
exit 1
|
|
fi
|
|
|
|
# Validate JSON syntax
|
|
if ! jq empty "$SCHEMA_PATH" 2>/dev/null; then
|
|
echo "::error::Schema for $SKILL is not valid JSON"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Schema is valid JSON"
|
|
|
|
- name: Run Validator Self-Test
|
|
id: validator
|
|
run: |
|
|
SKILL="${{ matrix.skill }}"
|
|
VALIDATOR="scripts/validate-skill.cjs"
|
|
CONFIG=".claude/skills/$SKILL/scripts/validate-config.json"
|
|
|
|
echo "=== Tier 3 Validator Self-Test: $SKILL ==="
|
|
|
|
if [ ! -f "$VALIDATOR" ]; then
|
|
echo "::error::Centralized validator 'scripts/validate-skill.cjs' not found"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "$CONFIG" ]; then
|
|
echo "::error::Tier 3 skill '$SKILL' missing required validate-config.json"
|
|
exit 1
|
|
fi
|
|
|
|
set +e
|
|
timeout 120 node "$VALIDATOR" "$SKILL" --self-test
|
|
EXIT_CODE=$?
|
|
set -e
|
|
|
|
case $EXIT_CODE in
|
|
0) echo "Validator self-test PASSED" ;;
|
|
2) echo "::warning::Validator skipped (missing optional tools)" ;;
|
|
124) echo "::error::Validator timed out"; exit 1 ;;
|
|
*) echo "::error::Validator failed (exit $EXIT_CODE)"; exit 1 ;;
|
|
esac
|
|
|
|
- name: Check Eval Suite Exists
|
|
run: |
|
|
SKILL="${{ matrix.skill }}"
|
|
EVAL_PATH=".claude/skills/$SKILL/evals/${SKILL}.yaml"
|
|
ALT_EVAL_PATH=".claude/skills/$SKILL/evals/${SKILL}.yml"
|
|
|
|
if [ ! -f "$EVAL_PATH" ] && [ ! -f "$ALT_EVAL_PATH" ]; then
|
|
echo "::error::Tier 3 skill '$SKILL' missing required eval suite"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Eval suite exists for $SKILL"
|
|
|
|
# ===========================================================================
|
|
# Job 3: Validate Tier 2 Skills (WARN ONLY)
|
|
# ===========================================================================
|
|
validate-tier2:
|
|
name: Tier 2 Validation (warn)
|
|
needs: detect-changes
|
|
if: needs.detect-changes.outputs.tier2_skills != '[]' && needs.detect-changes.outputs.tier2_skills != ''
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
continue-on-error: true # Don't block PR
|
|
strategy:
|
|
matrix:
|
|
skill: ${{ fromJson(needs.detect-changes.outputs.tier2_skills) }}
|
|
fail-fast: false
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup tools
|
|
run: |
|
|
npm install -g ajv-cli ajv-formats
|
|
sudo apt-get update && sudo apt-get install -y jq bc
|
|
|
|
- name: Validate Schema
|
|
continue-on-error: true
|
|
run: |
|
|
SKILL="${{ matrix.skill }}"
|
|
SCHEMA_PATH=".claude/skills/$SKILL/schemas/output.json"
|
|
|
|
echo "=== Tier 2 Schema Validation: $SKILL ==="
|
|
|
|
if [ ! -f "$SCHEMA_PATH" ]; then
|
|
echo "::warning::Tier 2 skill '$SKILL' missing schema"
|
|
exit 0
|
|
fi
|
|
|
|
if ! jq empty "$SCHEMA_PATH" 2>/dev/null; then
|
|
echo "::warning::Schema for $SKILL is not valid JSON"
|
|
exit 0
|
|
fi
|
|
|
|
echo "Schema valid"
|
|
|
|
- name: Run Validator Self-Test
|
|
continue-on-error: true
|
|
run: |
|
|
SKILL="${{ matrix.skill }}"
|
|
VALIDATOR="scripts/validate-skill.cjs"
|
|
CONFIG=".claude/skills/$SKILL/scripts/validate-config.json"
|
|
|
|
echo "=== Tier 2 Validator Self-Test: $SKILL ==="
|
|
|
|
if [ ! -f "$VALIDATOR" ] || [ ! -f "$CONFIG" ]; then
|
|
echo "::warning::Tier 2 skill '$SKILL' missing validator or config"
|
|
exit 0
|
|
fi
|
|
|
|
timeout 120 node "$VALIDATOR" "$SKILL" --self-test || echo "::warning::Validator failed for $SKILL"
|
|
|
|
# ===========================================================================
|
|
# Job 4: Validate Tier 1 Skills (WARN ONLY)
|
|
# ===========================================================================
|
|
validate-tier1:
|
|
name: Tier 1 Validation (warn)
|
|
needs: detect-changes
|
|
if: needs.detect-changes.outputs.tier1_skills != '[]' && needs.detect-changes.outputs.tier1_skills != ''
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
continue-on-error: true # Don't block PR
|
|
strategy:
|
|
matrix:
|
|
skill: ${{ fromJson(needs.detect-changes.outputs.tier1_skills) }}
|
|
fail-fast: false
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Validate Schema
|
|
continue-on-error: true
|
|
run: |
|
|
SKILL="${{ matrix.skill }}"
|
|
SCHEMA_PATH=".claude/skills/$SKILL/schemas/output.json"
|
|
|
|
echo "=== Tier 1 Schema Validation: $SKILL ==="
|
|
|
|
if [ ! -f "$SCHEMA_PATH" ]; then
|
|
echo "::warning::Tier 1 skill '$SKILL' missing schema"
|
|
exit 0
|
|
fi
|
|
|
|
if ! jq empty "$SCHEMA_PATH" 2>/dev/null; then
|
|
echo "::warning::Schema for $SKILL is not valid JSON"
|
|
else
|
|
echo "Schema valid"
|
|
fi
|
|
|
|
# ===========================================================================
|
|
# Job 5: Run Evaluation Suites (Manual Only)
|
|
# ===========================================================================
|
|
run-evals:
|
|
name: Evaluation Suite
|
|
needs: [detect-changes, validate-tier3]
|
|
if: |
|
|
always() &&
|
|
needs.detect-changes.outputs.has_changes == 'true' &&
|
|
github.event_name == 'workflow_dispatch' &&
|
|
inputs.run_evals == true
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
strategy:
|
|
matrix:
|
|
skill: ${{ fromJson(needs.detect-changes.outputs.tier3_skills) }}
|
|
fail-fast: false
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: ${{ env.NODE_VERSION }}
|
|
cache: 'npm'
|
|
|
|
- name: Install dependencies
|
|
run: npm ci
|
|
|
|
- name: Run Evaluation
|
|
env:
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
run: |
|
|
SKILL="${{ matrix.skill }}"
|
|
|
|
if [ -z "$ANTHROPIC_API_KEY" ]; then
|
|
echo "::warning::ANTHROPIC_API_KEY not set - skipping evals"
|
|
exit 0
|
|
fi
|
|
|
|
npx tsx scripts/run-skill-eval.ts --skill "$SKILL" --model claude-sonnet-4-6
|
|
|
|
# ===========================================================================
|
|
# Job 6: Summary Report
|
|
# ===========================================================================
|
|
report:
|
|
name: Validation Summary
|
|
needs: [detect-changes, validate-tier3, validate-tier2, validate-tier1]
|
|
if: always() && github.event_name == 'pull_request'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
|
|
steps:
|
|
- name: Generate Report
|
|
run: |
|
|
cat > report.md << 'EOF'
|
|
## Skill Validation Report
|
|
|
|
| Tier | Skills | Validation | Status |
|
|
|------|--------|------------|--------|
|
|
| **Tier 3** (Verified) | Blocks PR | Schema + Validator + Eval | ${{ needs.validate-tier3.result || 'skipped' }} |
|
|
| **Tier 2** (Validated) | Warn only | Schema + Validator | ${{ needs.validate-tier2.result || 'skipped' }} |
|
|
| **Tier 1** (Structured) | Warn only | Schema | ${{ needs.validate-tier1.result || 'skipped' }} |
|
|
| **Tier 0** (Advisory) | Skipped | None | - |
|
|
|
|
---
|
|
*Only Tier 3 failures block merge*
|
|
EOF
|
|
|
|
cat report.md
|
|
|
|
- name: Comment on PR
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const fs = require('fs');
|
|
const report = fs.readFileSync('report.md', 'utf8');
|
|
|
|
const { data: comments } = await github.rest.issues.listComments({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
issue_number: context.issue.number,
|
|
});
|
|
|
|
const botComment = comments.find(c =>
|
|
c.user?.type === 'Bot' &&
|
|
c.body?.includes('Skill Validation Report')
|
|
);
|
|
|
|
if (botComment) {
|
|
await github.rest.issues.updateComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
comment_id: botComment.id,
|
|
body: report
|
|
});
|
|
} else {
|
|
await github.rest.issues.createComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
issue_number: context.issue.number,
|
|
body: report
|
|
});
|
|
}
|
|
|
|
- name: Check Tier 3 Status
|
|
if: needs.validate-tier3.result == 'failure'
|
|
run: |
|
|
echo "::error::Tier 3 skill validation failed - blocking merge"
|
|
exit 1
|