mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
99511738c1
* fix(v3.4.1): bundle missing dependencies and test timeouts - Fixed issue #219: MCP bundle failing due to missing packages - Changed build scripts from --packages=external to selective externalization - Pure JS deps (fast-json-patch, jose, uuid) now bundled inline - Native modules properly externalized (better-sqlite3, hnswlib-node, etc.) - CommonJS modules with dynamic requires externalized (typescript, fast-glob, yaml, commander, cli-progress, ora) - Bundle size reduced from ~15MB to ~5MB - Added timeouts to handleQualityAssess tests to prevent CI failures - Updated version to 3.4.1 Closes #219 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * feat(v3.4.2): add skill validation system with trust tiers (ADR-056) Implement 4-layer skill validation system for deterministic outputs: - Tier 3 (Verified): 46 skills with full eval test suites - Tier 2 (Validated): 7 skills with executable validators - Tier 1 (Structured): 5 skills with JSON output schemas - Tier 0 (Advisory): 5 skills with SKILL.md guidance only Key additions: - JSON Schema validation templates for skill outputs - Executable validator scripts for correctness verification - Evaluation test framework with multi-model support - CLI commands: aqe skill report/summary/compare, aqe eval run/status - ReasoningBank integration for validation pattern learning - GitHub Actions workflow for CI skill validation - Trust tier frontmatter in all 63 skill SKILL.md files - Comprehensive documentation and user guides Infrastructure: - .claude/skills/.validation/ - Templates and frameworks - .claude/skills/*/schemas/ - Output schemas per skill - .claude/skills/*/scripts/ - Validator scripts per skill - .claude/skills/*/evals/ - Test cases per skill Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * fix(ci): trust tier-based skill validation - Tier 3 (Verified): Blocks PR on validation failure - Tier 2 (Validated): Warns only, doesn't block - Tier 1 (Structured): Warns only, doesn't block - Tier 0 (Advisory): Skipped entirely Reads trust_tier from SKILL.md frontmatter to determine validation requirements. Only 46 Tier 3 skills block merge. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * fix(validators): remove local keyword from self-test blocks Shell script fix: 'local' can only be used inside functions, not in if/else blocks. Both accessibility-testing and compatibility-testing validators had 'local' declarations in their self-test sections that caused errors when run outside of a function context. - Remove local test_file from accessibility-testing self-test - Remove local test_file, browser_count, tier1_pass from compatibility-testing self-test - Sync fixes to v3/assets/skills/ Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * fix(validators): add --self-test support to n8n skill validators The 5 n8n skill validators were failing CI because they didn't handle the --self-test flag properly. Added self-test handling that: - Checks for required tools (jq) - Verifies schema file exists and is valid JSON - Returns exit 0 on success Fixed validators: - n8n-expression-testing - n8n-integration-testing-patterns - n8n-security-testing - n8n-trigger-testing-strategies - n8n-workflow-testing-fundamentals Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
200 lines
6.6 KiB
JSON
200 lines
6.6 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://agentic-qe.dev/schemas/skill-output-meta.schema.json",
|
|
"title": "AQE Skill Output Meta-Schema",
|
|
"description": "Meta-schema that validates skill-specific output schemas properly extend the base template. Use this to validate schemas in .claude/skills/{skill}/schemas/output.json",
|
|
"type": "object",
|
|
"required": ["$schema", "$id", "title", "type", "required", "properties", "$defs"],
|
|
"properties": {
|
|
"$schema": {
|
|
"type": "string",
|
|
"const": "https://json-schema.org/draft/2020-12/schema",
|
|
"description": "Must use JSON Schema Draft 2020-12"
|
|
},
|
|
"$id": {
|
|
"type": "string",
|
|
"pattern": "^https://agentic-qe\\.dev/schemas/skills/[a-z][a-z0-9-]+/output\\.json$",
|
|
"description": "Schema ID must follow pattern: https://agentic-qe.dev/schemas/skills/{skill-name}/output.json"
|
|
},
|
|
"title": {
|
|
"type": "string",
|
|
"pattern": "^[A-Z][a-zA-Z0-9 -]+ Skill Output Schema$",
|
|
"description": "Title must follow pattern: '{Skill Name} Skill Output Schema'"
|
|
},
|
|
"description": {
|
|
"type": "string",
|
|
"minLength": 20,
|
|
"maxLength": 500,
|
|
"description": "Description of what the skill output represents"
|
|
},
|
|
"type": {
|
|
"const": "object",
|
|
"description": "Root type must be object"
|
|
},
|
|
"required": {
|
|
"type": "array",
|
|
"contains": {
|
|
"enum": ["skillName", "version", "timestamp", "status", "trustTier", "output"]
|
|
},
|
|
"minContains": 6,
|
|
"description": "Must require all base template fields"
|
|
},
|
|
"properties": {
|
|
"type": "object",
|
|
"required": [
|
|
"skillName",
|
|
"version",
|
|
"timestamp",
|
|
"status",
|
|
"trustTier",
|
|
"output",
|
|
"metadata",
|
|
"validation"
|
|
],
|
|
"properties": {
|
|
"skillName": {
|
|
"$ref": "#/$defs/baseFieldReference",
|
|
"description": "Must reference or match base template skillName"
|
|
},
|
|
"version": {
|
|
"$ref": "#/$defs/baseFieldReference",
|
|
"description": "Must reference or match base template version"
|
|
},
|
|
"timestamp": {
|
|
"$ref": "#/$defs/baseFieldReference",
|
|
"description": "Must reference or match base template timestamp"
|
|
},
|
|
"status": {
|
|
"$ref": "#/$defs/baseFieldReference",
|
|
"description": "Must reference or match base template status"
|
|
},
|
|
"trustTier": {
|
|
"$ref": "#/$defs/baseFieldReference",
|
|
"description": "Must reference or match base template trustTier"
|
|
},
|
|
"output": {
|
|
"type": "object",
|
|
"required": ["type"],
|
|
"properties": {
|
|
"type": {
|
|
"const": "object"
|
|
},
|
|
"required": {
|
|
"type": "array",
|
|
"description": "Skill-specific required output fields"
|
|
},
|
|
"properties": {
|
|
"type": "object",
|
|
"description": "Skill-specific output properties (must include or reference base template properties)"
|
|
}
|
|
},
|
|
"description": "Output must be an object type with skill-specific properties"
|
|
},
|
|
"metadata": {
|
|
"$ref": "#/$defs/baseFieldReference",
|
|
"description": "Must reference or extend base template metadata"
|
|
},
|
|
"validation": {
|
|
"$ref": "#/$defs/baseFieldReference",
|
|
"description": "Must reference or extend base template validation"
|
|
}
|
|
},
|
|
"description": "Properties must include all base template fields"
|
|
},
|
|
"$defs": {
|
|
"type": "object",
|
|
"description": "Definitions section for reusable schema components",
|
|
"properties": {
|
|
"finding": {
|
|
"type": "object",
|
|
"description": "If overriding finding, must include base required fields"
|
|
},
|
|
"recommendation": {
|
|
"type": "object",
|
|
"description": "If overriding recommendation, must include base required fields"
|
|
}
|
|
}
|
|
},
|
|
"allOf": {
|
|
"type": "array",
|
|
"description": "Optional allOf for extending base template"
|
|
},
|
|
"$ref": {
|
|
"type": "string",
|
|
"description": "Optional $ref to base template"
|
|
}
|
|
},
|
|
"$defs": {
|
|
"baseFieldReference": {
|
|
"type": "object",
|
|
"description": "Field that references or extends base template field",
|
|
"oneOf": [
|
|
{
|
|
"properties": {
|
|
"$ref": {
|
|
"type": "string",
|
|
"pattern": ".*skill-output-template\\.json#"
|
|
}
|
|
},
|
|
"required": ["$ref"]
|
|
},
|
|
{
|
|
"properties": {
|
|
"type": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"required": ["type"]
|
|
}
|
|
]
|
|
},
|
|
"skillSpecificExtension": {
|
|
"type": "object",
|
|
"description": "Pattern for skill-specific extensions",
|
|
"properties": {
|
|
"allOf": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object"
|
|
},
|
|
"description": "Use allOf to extend base definitions"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"examples": [
|
|
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://agentic-qe.dev/schemas/skills/security-testing/output.json",
|
|
"title": "Security Testing Skill Output Schema",
|
|
"description": "Output schema for security-testing skill with OWASP Top 10 findings",
|
|
"type": "object",
|
|
"required": ["skillName", "version", "timestamp", "status", "trustTier", "output"],
|
|
"properties": {
|
|
"skillName": { "const": "security-testing" },
|
|
"version": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" },
|
|
"timestamp": { "type": "string", "format": "date-time" },
|
|
"status": { "enum": ["success", "partial", "failed", "skipped"] },
|
|
"trustTier": { "const": 3 },
|
|
"output": {
|
|
"type": "object",
|
|
"required": ["owaspFindings", "summary"],
|
|
"properties": {
|
|
"summary": { "type": "string" },
|
|
"owaspFindings": {
|
|
"type": "object",
|
|
"properties": {
|
|
"A01": { "type": "array" },
|
|
"A02": { "type": "array" }
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"metadata": { "type": "object" },
|
|
"validation": { "type": "object" }
|
|
},
|
|
"$defs": {}
|
|
}
|
|
]
|
|
}
|