mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Dragan Spiridonov
99511738c1
* fix(v3.4.1): bundle missing dependencies and test timeouts - Fixed issue #219: MCP bundle failing due to missing packages - Changed build scripts from --packages=external to selective externalization - Pure JS deps (fast-json-patch, jose, uuid) now bundled inline - Native modules properly externalized (better-sqlite3, hnswlib-node, etc.) - CommonJS modules with dynamic requires externalized (typescript, fast-glob, yaml, commander, cli-progress, ora) - Bundle size reduced from ~15MB to ~5MB - Added timeouts to handleQualityAssess tests to prevent CI failures - Updated version to 3.4.1 Closes #219 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * feat(v3.4.2): add skill validation system with trust tiers (ADR-056) Implement 4-layer skill validation system for deterministic outputs: - Tier 3 (Verified): 46 skills with full eval test suites - Tier 2 (Validated): 7 skills with executable validators - Tier 1 (Structured): 5 skills with JSON output schemas - Tier 0 (Advisory): 5 skills with SKILL.md guidance only Key additions: - JSON Schema validation templates for skill outputs - Executable validator scripts for correctness verification - Evaluation test framework with multi-model support - CLI commands: aqe skill report/summary/compare, aqe eval run/status - ReasoningBank integration for validation pattern learning - GitHub Actions workflow for CI skill validation - Trust tier frontmatter in all 63 skill SKILL.md files - Comprehensive documentation and user guides Infrastructure: - .claude/skills/.validation/ - Templates and frameworks - .claude/skills/*/schemas/ - Output schemas per skill - .claude/skills/*/scripts/ - Validator scripts per skill - .claude/skills/*/evals/ - Test cases per skill Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * fix(ci): trust tier-based skill validation - Tier 3 (Verified): Blocks PR on validation failure - Tier 2 (Validated): Warns only, doesn't block - Tier 1 (Structured): Warns only, doesn't block - Tier 0 (Advisory): Skipped entirely Reads trust_tier from SKILL.md frontmatter to determine validation requirements. Only 46 Tier 3 skills block merge. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * fix(validators): remove local keyword from self-test blocks Shell script fix: 'local' can only be used inside functions, not in if/else blocks. Both accessibility-testing and compatibility-testing validators had 'local' declarations in their self-test sections that caused errors when run outside of a function context. - Remove local test_file from accessibility-testing self-test - Remove local test_file, browser_count, tier1_pass from compatibility-testing self-test - Sync fixes to v3/assets/skills/ Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * fix(validators): add --self-test support to n8n skill validators The 5 n8n skill validators were failing CI because they didn't handle the --self-test flag properly. Added self-test handling that: - Checks for required tools (jq) - Verifies schema file exists and is valid JSON - Returns exit 0 on success Fixed validators: - n8n-expression-testing - n8n-integration-testing-patterns - n8n-security-testing - n8n-trigger-testing-strategies - n8n-workflow-testing-fundamentals Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
611 lines
19 KiB
JSON
611 lines
19 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://agentic-qe.dev/schemas/skill-output-template.json",
|
|
"title": "AQE Skill Output Template Schema",
|
|
"description": "Template schema for skill output validation. Extend this for specific skills. All validated skills must conform to this base structure.",
|
|
"type": "object",
|
|
"required": ["skillName", "version", "timestamp", "status", "trustTier", "output"],
|
|
"properties": {
|
|
"skillName": {
|
|
"type": "string",
|
|
"pattern": "^[a-z][a-z0-9-]*$",
|
|
"minLength": 2,
|
|
"maxLength": 64,
|
|
"description": "Name of the skill that produced this output (lowercase, hyphenated)"
|
|
},
|
|
"version": {
|
|
"type": "string",
|
|
"pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?$",
|
|
"description": "Semantic version of the skill (MAJOR.MINOR.PATCH with optional prerelease)"
|
|
},
|
|
"timestamp": {
|
|
"type": "string",
|
|
"format": "date-time",
|
|
"description": "ISO 8601 timestamp of output generation"
|
|
},
|
|
"status": {
|
|
"type": "string",
|
|
"enum": ["success", "partial", "failed", "skipped"],
|
|
"description": "Overall execution status: success=all checks passed, partial=some passed, failed=critical failure, skipped=not applicable"
|
|
},
|
|
"trustTier": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"maximum": 3,
|
|
"description": "Trust tier: 0=declarative only, 1=has schema, 2=has validator, 3=has eval suite"
|
|
},
|
|
"output": {
|
|
"type": "object",
|
|
"description": "Skill-specific output data. Extend in skill-specific schema.",
|
|
"required": [],
|
|
"properties": {
|
|
"summary": {
|
|
"type": "string",
|
|
"minLength": 10,
|
|
"maxLength": 2000,
|
|
"description": "Human-readable summary of findings (10-2000 chars)"
|
|
},
|
|
"score": {
|
|
"$ref": "#/$defs/score",
|
|
"description": "Overall score/grade if applicable"
|
|
},
|
|
"findings": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/$defs/finding"
|
|
},
|
|
"maxItems": 500,
|
|
"description": "List of findings/issues discovered"
|
|
},
|
|
"recommendations": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/$defs/recommendation"
|
|
},
|
|
"maxItems": 100,
|
|
"description": "Actionable recommendations for improvement"
|
|
},
|
|
"metrics": {
|
|
"$ref": "#/$defs/metrics",
|
|
"description": "Quantitative metrics from analysis"
|
|
},
|
|
"artifacts": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/$defs/artifact"
|
|
},
|
|
"maxItems": 50,
|
|
"description": "Generated artifacts (reports, files, etc.)"
|
|
},
|
|
"categories": {
|
|
"type": "object",
|
|
"additionalProperties": {
|
|
"$ref": "#/$defs/categoryScore"
|
|
},
|
|
"description": "Breakdown by category (e.g., OWASP categories, testability principles)"
|
|
},
|
|
"timeline": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/$defs/timelineEvent"
|
|
},
|
|
"description": "Execution timeline for debugging/auditing"
|
|
}
|
|
}
|
|
},
|
|
"metadata": {
|
|
"type": "object",
|
|
"description": "Execution metadata for debugging, caching, and auditing",
|
|
"properties": {
|
|
"executionTimeMs": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"maximum": 3600000,
|
|
"description": "Execution time in milliseconds (max 1 hour)"
|
|
},
|
|
"toolsUsed": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"pattern": "^[a-zA-Z][a-zA-Z0-9_-]*$"
|
|
},
|
|
"uniqueItems": true,
|
|
"maxItems": 50,
|
|
"description": "Tools/utilities invoked during execution"
|
|
},
|
|
"agentId": {
|
|
"type": "string",
|
|
"pattern": "^qe-[a-z][a-z0-9-]*$",
|
|
"description": "ID of the agent that executed the skill (e.g., qe-security-scanner)"
|
|
},
|
|
"modelUsed": {
|
|
"type": "string",
|
|
"description": "LLM model used for execution (e.g., claude-3.5-sonnet)"
|
|
},
|
|
"inputHash": {
|
|
"type": "string",
|
|
"pattern": "^[a-f0-9]{64}$",
|
|
"description": "SHA-256 hash of input for caching/deduplication"
|
|
},
|
|
"parentTaskId": {
|
|
"type": "string",
|
|
"format": "uuid",
|
|
"description": "UUID of parent task if part of swarm execution"
|
|
},
|
|
"targetUrl": {
|
|
"type": "string",
|
|
"format": "uri",
|
|
"description": "Target URL if applicable"
|
|
},
|
|
"targetPath": {
|
|
"type": "string",
|
|
"description": "Target file/directory path if applicable"
|
|
},
|
|
"environment": {
|
|
"type": "string",
|
|
"enum": ["development", "staging", "production", "ci"],
|
|
"description": "Environment where skill was executed"
|
|
},
|
|
"retryCount": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"maximum": 10,
|
|
"description": "Number of retries before success/failure"
|
|
}
|
|
}
|
|
},
|
|
"validation": {
|
|
"type": "object",
|
|
"description": "Self-reported validation status from skill validator",
|
|
"properties": {
|
|
"schemaValid": {
|
|
"type": "boolean",
|
|
"description": "Whether output passes JSON schema validation"
|
|
},
|
|
"contentValid": {
|
|
"type": "boolean",
|
|
"description": "Whether output passes content validation rules"
|
|
},
|
|
"confidence": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"description": "Confidence score for output correctness (0.0-1.0)"
|
|
},
|
|
"warnings": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"maxLength": 500
|
|
},
|
|
"maxItems": 20,
|
|
"description": "Validation warnings (non-fatal issues)"
|
|
},
|
|
"errors": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"maxLength": 500
|
|
},
|
|
"maxItems": 20,
|
|
"description": "Validation errors encountered"
|
|
},
|
|
"validatorVersion": {
|
|
"type": "string",
|
|
"pattern": "^\\d+\\.\\d+\\.\\d+$",
|
|
"description": "Version of validator used"
|
|
}
|
|
}
|
|
},
|
|
"learning": {
|
|
"type": "object",
|
|
"description": "Data for ReasoningBank learning integration",
|
|
"properties": {
|
|
"patternsDetected": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"maxLength": 200
|
|
},
|
|
"maxItems": 20,
|
|
"description": "Pattern names detected during execution"
|
|
},
|
|
"reward": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"description": "Reward signal for reinforcement learning (0.0-1.0)"
|
|
},
|
|
"feedbackLoop": {
|
|
"type": "object",
|
|
"properties": {
|
|
"previousRunId": {
|
|
"type": "string",
|
|
"format": "uuid",
|
|
"description": "ID of previous run for comparison"
|
|
},
|
|
"improvement": {
|
|
"type": "number",
|
|
"minimum": -1,
|
|
"maximum": 1,
|
|
"description": "Improvement over previous run (-1.0 to 1.0)"
|
|
}
|
|
},
|
|
"description": "Quality feedback loop data"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"$defs": {
|
|
"score": {
|
|
"type": "object",
|
|
"description": "Standardized score object",
|
|
"required": ["value", "max"],
|
|
"properties": {
|
|
"value": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"description": "Score value"
|
|
},
|
|
"max": {
|
|
"type": "number",
|
|
"minimum": 1,
|
|
"description": "Maximum possible score"
|
|
},
|
|
"grade": {
|
|
"type": "string",
|
|
"pattern": "^[A-F][+-]?$",
|
|
"description": "Letter grade (A, A-, B+, B, etc.)"
|
|
},
|
|
"percentile": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 100,
|
|
"description": "Percentile ranking if available"
|
|
},
|
|
"trend": {
|
|
"type": "string",
|
|
"enum": ["improving", "stable", "declining", "unknown"],
|
|
"description": "Trend compared to previous assessments"
|
|
}
|
|
}
|
|
},
|
|
"finding": {
|
|
"type": "object",
|
|
"description": "Individual finding/issue discovered",
|
|
"required": ["id", "title", "severity"],
|
|
"properties": {
|
|
"id": {
|
|
"type": "string",
|
|
"pattern": "^[A-Z]{2,6}-\\d{3,6}$",
|
|
"description": "Unique finding identifier (e.g., SEC-001, A11Y-0042)"
|
|
},
|
|
"title": {
|
|
"type": "string",
|
|
"minLength": 5,
|
|
"maxLength": 200,
|
|
"description": "Finding title (5-200 chars)"
|
|
},
|
|
"description": {
|
|
"type": "string",
|
|
"maxLength": 2000,
|
|
"description": "Detailed description"
|
|
},
|
|
"severity": {
|
|
"type": "string",
|
|
"enum": ["critical", "high", "medium", "low", "info"],
|
|
"description": "Severity: critical=immediate fix, high=fix soon, medium=plan fix, low=nice to have, info=informational"
|
|
},
|
|
"category": {
|
|
"type": "string",
|
|
"maxLength": 100,
|
|
"description": "Finding category (e.g., A01-Broken-Access-Control, WCAG-1.1.1)"
|
|
},
|
|
"location": {
|
|
"$ref": "#/$defs/location",
|
|
"description": "Location of the finding"
|
|
},
|
|
"evidence": {
|
|
"type": "string",
|
|
"maxLength": 5000,
|
|
"description": "Evidence supporting the finding (code snippet, screenshot path, etc.)"
|
|
},
|
|
"cwe": {
|
|
"type": "string",
|
|
"pattern": "^CWE-\\d{1,4}$",
|
|
"description": "CWE identifier (e.g., CWE-79 for XSS)"
|
|
},
|
|
"cvss": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 10,
|
|
"description": "CVSS v3.1 score (0.0-10.0)"
|
|
},
|
|
"owasp": {
|
|
"type": "string",
|
|
"pattern": "^A\\d{2}:\\d{4}$",
|
|
"description": "OWASP Top 10 category (e.g., A01:2021)"
|
|
},
|
|
"wcag": {
|
|
"type": "string",
|
|
"pattern": "^\\d+\\.\\d+\\.\\d+$",
|
|
"description": "WCAG success criterion (e.g., 1.1.1)"
|
|
},
|
|
"remediation": {
|
|
"type": "string",
|
|
"maxLength": 2000,
|
|
"description": "How to fix this finding"
|
|
},
|
|
"falsePositive": {
|
|
"type": "boolean",
|
|
"default": false,
|
|
"description": "Whether this is a potential false positive"
|
|
},
|
|
"confidence": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"description": "Confidence in this finding (0.0-1.0)"
|
|
}
|
|
}
|
|
},
|
|
"recommendation": {
|
|
"type": "object",
|
|
"description": "Actionable recommendation for improvement",
|
|
"required": ["id", "title", "priority"],
|
|
"properties": {
|
|
"id": {
|
|
"type": "string",
|
|
"pattern": "^REC-\\d{3,6}$",
|
|
"description": "Unique recommendation identifier (e.g., REC-001)"
|
|
},
|
|
"title": {
|
|
"type": "string",
|
|
"minLength": 5,
|
|
"maxLength": 200,
|
|
"description": "Recommendation title"
|
|
},
|
|
"description": {
|
|
"type": "string",
|
|
"maxLength": 2000,
|
|
"description": "Detailed recommendation"
|
|
},
|
|
"priority": {
|
|
"type": "string",
|
|
"enum": ["critical", "high", "medium", "low"],
|
|
"description": "Priority level"
|
|
},
|
|
"effort": {
|
|
"type": "string",
|
|
"enum": ["trivial", "low", "medium", "high", "major"],
|
|
"description": "Estimated effort: trivial=<1hr, low=1-4hr, medium=1-3d, high=1-2wk, major=>2wk"
|
|
},
|
|
"impact": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"maximum": 10,
|
|
"description": "Expected impact score (1-10)"
|
|
},
|
|
"relatedFindings": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"pattern": "^[A-Z]{2,6}-\\d{3,6}$"
|
|
},
|
|
"description": "IDs of related findings"
|
|
},
|
|
"codeExample": {
|
|
"type": "string",
|
|
"maxLength": 5000,
|
|
"description": "Example code for remediation"
|
|
},
|
|
"resources": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"required": ["title", "url"],
|
|
"properties": {
|
|
"title": { "type": "string" },
|
|
"url": { "type": "string", "format": "uri" }
|
|
}
|
|
},
|
|
"maxItems": 10,
|
|
"description": "External resources for learning more"
|
|
}
|
|
}
|
|
},
|
|
"artifact": {
|
|
"type": "object",
|
|
"description": "Generated artifact (file, report, etc.)",
|
|
"required": ["type", "path"],
|
|
"properties": {
|
|
"type": {
|
|
"type": "string",
|
|
"enum": ["report", "test", "config", "data", "log", "screenshot", "video", "coverage"],
|
|
"description": "Artifact type"
|
|
},
|
|
"path": {
|
|
"type": "string",
|
|
"maxLength": 500,
|
|
"description": "Path to artifact (relative to project root)"
|
|
},
|
|
"format": {
|
|
"type": "string",
|
|
"enum": ["json", "html", "md", "txt", "csv", "xml", "yaml", "png", "jpg", "webm", "mp4"],
|
|
"description": "Artifact format"
|
|
},
|
|
"description": {
|
|
"type": "string",
|
|
"maxLength": 500,
|
|
"description": "Artifact description"
|
|
},
|
|
"sizeBytes": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "File size in bytes"
|
|
},
|
|
"checksum": {
|
|
"type": "string",
|
|
"pattern": "^sha256:[a-f0-9]{64}$",
|
|
"description": "SHA-256 checksum for integrity verification"
|
|
}
|
|
}
|
|
},
|
|
"location": {
|
|
"type": "object",
|
|
"description": "Location reference for findings",
|
|
"properties": {
|
|
"file": {
|
|
"type": "string",
|
|
"maxLength": 500,
|
|
"description": "File path"
|
|
},
|
|
"line": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "Line number"
|
|
},
|
|
"column": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "Column number"
|
|
},
|
|
"endLine": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "End line number for multi-line findings"
|
|
},
|
|
"endColumn": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "End column number"
|
|
},
|
|
"url": {
|
|
"type": "string",
|
|
"format": "uri",
|
|
"description": "URL if web-based finding"
|
|
},
|
|
"selector": {
|
|
"type": "string",
|
|
"maxLength": 500,
|
|
"description": "CSS selector for DOM element"
|
|
},
|
|
"xpath": {
|
|
"type": "string",
|
|
"maxLength": 500,
|
|
"description": "XPath for DOM element"
|
|
}
|
|
}
|
|
},
|
|
"metrics": {
|
|
"type": "object",
|
|
"description": "Quantitative metrics from analysis",
|
|
"properties": {
|
|
"total": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Total items analyzed"
|
|
},
|
|
"passed": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Items that passed"
|
|
},
|
|
"failed": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Items that failed"
|
|
},
|
|
"skipped": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Items skipped"
|
|
},
|
|
"coverage": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 100,
|
|
"description": "Coverage percentage"
|
|
},
|
|
"duration": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Duration in milliseconds"
|
|
},
|
|
"custom": {
|
|
"type": "object",
|
|
"additionalProperties": {
|
|
"oneOf": [
|
|
{ "type": "number" },
|
|
{ "type": "string" },
|
|
{ "type": "boolean" }
|
|
]
|
|
},
|
|
"description": "Skill-specific custom metrics"
|
|
}
|
|
}
|
|
},
|
|
"categoryScore": {
|
|
"type": "object",
|
|
"description": "Score for a specific category",
|
|
"required": ["score"],
|
|
"properties": {
|
|
"score": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 100,
|
|
"description": "Category score (0-100)"
|
|
},
|
|
"weight": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 1,
|
|
"description": "Weight in overall score (0.0-1.0)"
|
|
},
|
|
"description": {
|
|
"type": "string",
|
|
"maxLength": 500,
|
|
"description": "Category description"
|
|
},
|
|
"grade": {
|
|
"type": "string",
|
|
"pattern": "^[A-F][+-]?$",
|
|
"description": "Letter grade for this category"
|
|
},
|
|
"findingCount": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Number of findings in this category"
|
|
}
|
|
}
|
|
},
|
|
"timelineEvent": {
|
|
"type": "object",
|
|
"description": "Event in execution timeline",
|
|
"required": ["timestamp", "event"],
|
|
"properties": {
|
|
"timestamp": {
|
|
"type": "string",
|
|
"format": "date-time",
|
|
"description": "When the event occurred"
|
|
},
|
|
"event": {
|
|
"type": "string",
|
|
"maxLength": 200,
|
|
"description": "Event description"
|
|
},
|
|
"type": {
|
|
"type": "string",
|
|
"enum": ["start", "checkpoint", "warning", "error", "complete"],
|
|
"description": "Event type"
|
|
},
|
|
"durationMs": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Duration since previous event"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|