mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Profa
545412f863
Scores SKILL.md files across 8 weighted dimensions (description, frontmatter, conciseness, actionability, structure, triggers, code examples, metadata) and assigns letter grades A-F. Supports --skill, --suggestions, --json, and --min-score filters. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
512 lines
19 KiB
TypeScript
512 lines
19 KiB
TypeScript
#!/usr/bin/env npx tsx
|
|
/**
|
|
* Skill Quality Scorer
|
|
* Deterministic quality scoring for SKILL.md files across 8 dimensions.
|
|
*
|
|
* Usage:
|
|
* npx tsx scripts/score-skill-quality.ts
|
|
* npx tsx scripts/score-skill-quality.ts --skill api-testing-patterns
|
|
* npx tsx scripts/score-skill-quality.ts --suggestions
|
|
* npx tsx scripts/score-skill-quality.ts --json
|
|
* npx tsx scripts/score-skill-quality.ts --min-score 70
|
|
*/
|
|
|
|
import { readFileSync, writeFileSync, existsSync, readdirSync, statSync } from 'fs';
|
|
import { join, dirname } from 'path';
|
|
|
|
// ============================================================================
|
|
// TYPES
|
|
// ============================================================================
|
|
|
|
interface DimensionScores {
|
|
description: number; // 0-10
|
|
frontmatter: number; // 0-10
|
|
conciseness: number; // 0-10
|
|
actionability: number; // 0-10
|
|
structure: number; // 0-10
|
|
triggers: number; // 0-10
|
|
codeExamples: number; // 0-10
|
|
metadata: number; // 0-10
|
|
}
|
|
|
|
interface SkillScore {
|
|
name: string;
|
|
score: number;
|
|
grade: string;
|
|
dimensions: DimensionScores;
|
|
suggestions: string[];
|
|
}
|
|
|
|
// ============================================================================
|
|
// CONSTANTS
|
|
// ============================================================================
|
|
|
|
const SKILLS_DIR = '.claude/skills';
|
|
const PLATFORM_PREFIXES = ['v3-', 'flow-nexus-', 'agentdb-', 'reasoningbank-', 'swarm-'];
|
|
|
|
const EXPECTED_FRONTMATTER_FIELDS = [
|
|
'name', 'description', 'category', 'priority', 'tokenEstimate',
|
|
'agents', 'tags', 'trust_tier', 'validation', 'dependencies',
|
|
'implementation_status', 'optimization_version', 'last_optimized',
|
|
'quick_reference_card',
|
|
];
|
|
|
|
const TRIGGER_TERMS = [
|
|
'owasp', 'pact', 'k6', 'artillery', 'jmeter', 'wcag', 'gdpr', 'hipaa',
|
|
'soc2', 'pci', 'jest', 'vitest', 'playwright', 'cypress', 'selenium',
|
|
'supertest', 'graphql', 'rest', 'grpc', 'openapi', 'swagger', 'postman',
|
|
'cucumber', 'gherkin', 'bdd', 'tdd', 'mutation', 'stryker', 'pitest',
|
|
'sonarqube', 'eslint', 'prettier', 'docker', 'kubernetes', 'terraform',
|
|
'kafka', 'rabbitmq', 'redis', 'postgresql', 'mongodb', 'dynamodb',
|
|
'oauth', 'jwt', 'saml', 'oidc', 'xss', 'sqli', 'csrf', 'ssrf',
|
|
'sast', 'dast', 'iast', 'sca', 'sbom', 'cve',
|
|
];
|
|
|
|
const WEIGHTS: Record<keyof DimensionScores, number> = {
|
|
description: 20,
|
|
frontmatter: 15,
|
|
conciseness: 15,
|
|
actionability: 15,
|
|
structure: 15,
|
|
triggers: 10,
|
|
codeExamples: 5,
|
|
metadata: 5,
|
|
};
|
|
|
|
// ============================================================================
|
|
// FRONTMATTER PARSER (from detect-skill-conflicts.ts)
|
|
// ============================================================================
|
|
|
|
function parseYamlFrontmatter(content: string): Record<string, unknown> {
|
|
const match = content.match(/^---\n([\s\S]*?)\n---/);
|
|
if (!match) return {};
|
|
|
|
const result: Record<string, unknown> = {};
|
|
const lines = match[1].split('\n');
|
|
let inNested = false;
|
|
let nestedKey = '';
|
|
const nestedObj: Record<string, unknown> = {};
|
|
|
|
for (const line of lines) {
|
|
if (!line.trim() || line.trim().startsWith('#')) continue;
|
|
const indent = line.search(/\S/);
|
|
|
|
if (indent > 0 && inNested) {
|
|
const kv = line.trim().match(/^([\w_]+):\s*(.+)$/);
|
|
if (kv) nestedObj[kv[1]] = parseYamlValue(kv[2]);
|
|
continue;
|
|
}
|
|
if (indent === 0 && inNested) {
|
|
result[nestedKey] = { ...nestedObj };
|
|
inNested = false;
|
|
}
|
|
|
|
const kv = line.trim().match(/^([\w_]+):\s*(.*)$/);
|
|
if (kv) {
|
|
const [, key, value] = kv;
|
|
if (!value || value.trim() === '') {
|
|
inNested = true;
|
|
nestedKey = key;
|
|
Object.keys(nestedObj).forEach(k => delete nestedObj[k]);
|
|
} else {
|
|
result[key] = parseYamlValue(value.trim());
|
|
}
|
|
}
|
|
}
|
|
if (inNested) result[nestedKey] = { ...nestedObj };
|
|
return result;
|
|
}
|
|
|
|
function parseYamlValue(value: string): unknown {
|
|
const t = value.trim();
|
|
if ((t.startsWith('"') && t.endsWith('"')) || (t.startsWith("'") && t.endsWith("'")))
|
|
return t.slice(1, -1);
|
|
if (t.startsWith('[') && t.endsWith(']'))
|
|
return t.slice(1, -1).split(',').map(i => {
|
|
const s = i.trim();
|
|
return (s.startsWith('"') || s.startsWith("'")) ? s.slice(1, -1) : s;
|
|
});
|
|
if (t === 'true') return true;
|
|
if (t === 'false') return false;
|
|
const n = Number(t);
|
|
if (!isNaN(n) && t !== '') return n;
|
|
return t;
|
|
}
|
|
|
|
// ============================================================================
|
|
// SKILL DISCOVERY (from detect-skill-conflicts.ts)
|
|
// ============================================================================
|
|
|
|
function getProjectRoot(): string {
|
|
let dir = process.cwd();
|
|
while (dir !== '/') {
|
|
if (existsSync(join(dir, 'package.json'))) return dir;
|
|
dir = dirname(dir);
|
|
}
|
|
return process.cwd();
|
|
}
|
|
|
|
interface SkillFile {
|
|
name: string;
|
|
content: string;
|
|
frontmatter: Record<string, unknown>;
|
|
body: string;
|
|
}
|
|
|
|
function discoverSkills(projectRoot: string, filterSkill?: string): SkillFile[] {
|
|
const skillsPath = join(projectRoot, SKILLS_DIR);
|
|
if (!existsSync(skillsPath)) {
|
|
console.error(`Skills directory not found: ${skillsPath}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
const skills: SkillFile[] = [];
|
|
for (const entry of readdirSync(skillsPath)) {
|
|
if (entry.startsWith('.') || !statSync(join(skillsPath, entry)).isDirectory()) continue;
|
|
if (PLATFORM_PREFIXES.some(p => entry.startsWith(p))) continue;
|
|
if (filterSkill && entry !== filterSkill) continue;
|
|
|
|
const mdPath = existsSync(join(skillsPath, entry, 'SKILL.md'))
|
|
? join(skillsPath, entry, 'SKILL.md')
|
|
: existsSync(join(skillsPath, entry, 'skill.md'))
|
|
? join(skillsPath, entry, 'skill.md')
|
|
: null;
|
|
if (!mdPath) continue;
|
|
|
|
const content = readFileSync(mdPath, 'utf-8');
|
|
const frontmatter = parseYamlFrontmatter(content);
|
|
const bodyMatch = content.match(/^---[\s\S]*?---\s*\n([\s\S]*)$/);
|
|
const body = bodyMatch ? bodyMatch[1] : content;
|
|
|
|
skills.push({ name: entry, content, frontmatter, body });
|
|
}
|
|
|
|
return skills.sort((a, b) => a.name.localeCompare(b.name));
|
|
}
|
|
|
|
// ============================================================================
|
|
// SCORING DIMENSIONS
|
|
// ============================================================================
|
|
|
|
function scoreDescription(fm: Record<string, unknown>, body: string): { score: number; suggestions: string[] } {
|
|
let pts = 0;
|
|
const suggestions: string[] = [];
|
|
const desc = String(fm.description || '');
|
|
|
|
if (!desc) {
|
|
suggestions.push('Add a "description" field to frontmatter');
|
|
return { score: 0, suggestions };
|
|
}
|
|
|
|
// Length sweet spot: 50-200 chars
|
|
if (desc.length >= 50 && desc.length <= 200) pts += 4;
|
|
else if (desc.length >= 30 && desc.length <= 250) pts += 2;
|
|
else suggestions.push(`Description length ${desc.length} chars — aim for 50-200`);
|
|
|
|
// "Use when" trigger phrase
|
|
if (/use when/i.test(desc)) pts += 3;
|
|
else suggestions.push('Add "Use when..." trigger phrase to description');
|
|
|
|
// Named tools/standards in description
|
|
const descLower = desc.toLowerCase();
|
|
const mentionedTools = TRIGGER_TERMS.filter(t => descLower.includes(t));
|
|
if (mentionedTools.length >= 2) pts += 3;
|
|
else if (mentionedTools.length === 1) pts += 2;
|
|
else suggestions.push('Mention specific tools/standards in description (e.g., Pact, k6, OWASP)');
|
|
|
|
return { score: Math.min(10, pts), suggestions };
|
|
}
|
|
|
|
function scoreFrontmatter(fm: Record<string, unknown>): { score: number; suggestions: string[] } {
|
|
const suggestions: string[] = [];
|
|
const present = EXPECTED_FRONTMATTER_FIELDS.filter(f => fm[f] !== undefined);
|
|
const ratio = present.length / EXPECTED_FRONTMATTER_FIELDS.length;
|
|
const score = Math.round(ratio * 10);
|
|
|
|
const missing = EXPECTED_FRONTMATTER_FIELDS.filter(f => fm[f] === undefined);
|
|
if (missing.length > 0 && missing.length <= 5) {
|
|
suggestions.push(`Add missing frontmatter: ${missing.join(', ')}`);
|
|
} else if (missing.length > 5) {
|
|
suggestions.push(`Add ${missing.length} missing frontmatter fields (e.g., ${missing.slice(0, 3).join(', ')})`);
|
|
}
|
|
|
|
return { score, suggestions };
|
|
}
|
|
|
|
function scoreConciseness(body: string): { score: number; suggestions: string[] } {
|
|
const suggestions: string[] = [];
|
|
const lines = body.split('\n').length;
|
|
|
|
let score: number;
|
|
if (lines >= 200 && lines <= 400) score = 10;
|
|
else if (lines >= 150 && lines <= 500) score = 8;
|
|
else if (lines >= 100 && lines <= 600) score = 6;
|
|
else if (lines >= 50 && lines <= 800) score = 4;
|
|
else if (lines > 1200) { score = 1; suggestions.push(`Body is ${lines} lines — aim for 200-400 for optimal readability`); }
|
|
else if (lines > 800) { score = 3; suggestions.push(`Body is ${lines} lines — consider trimming to 200-400`); }
|
|
else if (lines < 50) { score = 3; suggestions.push(`Body is only ${lines} lines — expand with tables, examples, decision matrices`); }
|
|
else score = 4;
|
|
|
|
return { score, suggestions };
|
|
}
|
|
|
|
function scoreActionability(body: string): { score: number; suggestions: string[] } {
|
|
let pts = 0;
|
|
const suggestions: string[] = [];
|
|
|
|
// <default_to_action> block
|
|
if (body.includes('<default_to_action>')) pts += 5;
|
|
else suggestions.push('Add a <default_to_action> block with immediate action steps');
|
|
|
|
// Verb-led numbered steps (e.g., "1. IDENTIFY", "2. TEST")
|
|
const numberedSteps = body.match(/^\d+\.\s+[A-Z]{2,}/gm);
|
|
if (numberedSteps && numberedSteps.length >= 3) pts += 3;
|
|
else if (numberedSteps && numberedSteps.length >= 1) pts += 1;
|
|
else suggestions.push('Use verb-led numbered steps (e.g., "1. IDENTIFY...", "2. TEST...")');
|
|
|
|
// Decision matrix (pattern selection, when-to-use table)
|
|
if (/quick pattern selection|when to use|decision matrix|pattern selection/i.test(body)) pts += 2;
|
|
else if (/\|.*\|.*\|/m.test(body)) pts += 1;
|
|
|
|
return { score: Math.min(10, pts), suggestions };
|
|
}
|
|
|
|
function scoreStructure(body: string): { score: number; suggestions: string[] } {
|
|
let pts = 0;
|
|
const suggestions: string[] = [];
|
|
|
|
// Quick Reference Card
|
|
if (/quick reference card/i.test(body)) pts += 3;
|
|
else suggestions.push('Add a "Quick Reference Card" section');
|
|
|
|
// Related Skills / Integration section
|
|
if (/related skills|related commands|integration with/i.test(body)) pts += 2;
|
|
else suggestions.push('Add a "Related Skills" section');
|
|
|
|
// Remember / Rules / Best Practices
|
|
if (/^##?\s*(remember|rules|best practices|critical)/im.test(body)) pts += 2;
|
|
|
|
// 3+ markdown tables
|
|
const tables = body.match(/\|.*\|.*\|/g);
|
|
if (tables && tables.length >= 6) pts += 2; // header+row = 2 lines per table, so 6 = 3+ tables
|
|
else if (tables && tables.length >= 4) pts += 1;
|
|
else suggestions.push('Add more markdown tables for quick-scan reference');
|
|
|
|
// Agent Coordination section
|
|
if (/agent coordination/i.test(body)) pts += 1;
|
|
|
|
return { score: Math.min(10, pts), suggestions };
|
|
}
|
|
|
|
function scoreTriggers(fm: Record<string, unknown>, body: string): { score: number; suggestions: string[] } {
|
|
const suggestions: string[] = [];
|
|
const desc = String(fm.description || '').toLowerCase();
|
|
const first60 = body.split('\n').slice(0, 60).join('\n').toLowerCase();
|
|
const combined = desc + ' ' + first60;
|
|
|
|
const found = TRIGGER_TERMS.filter(t => combined.includes(t));
|
|
let score: number;
|
|
if (found.length >= 5) score = 10;
|
|
else if (found.length >= 3) score = 7;
|
|
else if (found.length >= 1) score = 4;
|
|
else { score = 1; suggestions.push('Add specific tool/standard names (OWASP, Pact, k6, WCAG, etc.) in description and intro'); }
|
|
|
|
return { score, suggestions };
|
|
}
|
|
|
|
function scoreCodeExamples(body: string): { score: number; suggestions: string[] } {
|
|
const suggestions: string[] = [];
|
|
const blocks = body.match(/```[\s\S]*?```/g);
|
|
const count = blocks ? blocks.length : 0;
|
|
|
|
let score: number;
|
|
if (count >= 4) score = 10;
|
|
else if (count >= 2) score = 7;
|
|
else if (count === 1) score = 4;
|
|
else { score = 0; suggestions.push('Add fenced code examples to illustrate usage'); }
|
|
|
|
return { score, suggestions };
|
|
}
|
|
|
|
function scoreMetadata(fm: Record<string, unknown>): { score: number; suggestions: string[] } {
|
|
let pts = 0;
|
|
const suggestions: string[] = [];
|
|
|
|
// Tags (3+)
|
|
const tags = Array.isArray(fm.tags) ? fm.tags : [];
|
|
if (tags.length >= 3) pts += 3;
|
|
else if (tags.length >= 1) pts += 1;
|
|
else suggestions.push('Add 3+ tags to frontmatter');
|
|
|
|
// Valid category
|
|
if (fm.category) pts += 3;
|
|
else suggestions.push('Add a "category" field to frontmatter');
|
|
|
|
// Priority set
|
|
if (fm.priority) pts += 2;
|
|
else suggestions.push('Add a "priority" field to frontmatter');
|
|
|
|
// tokenEstimate present
|
|
if (fm.tokenEstimate !== undefined) pts += 2;
|
|
else suggestions.push('Add a "tokenEstimate" field to frontmatter');
|
|
|
|
return { score: Math.min(10, pts), suggestions };
|
|
}
|
|
|
|
// ============================================================================
|
|
// SCORING ORCHESTRATION
|
|
// ============================================================================
|
|
|
|
function scoreSkill(skill: SkillFile): SkillScore {
|
|
const allSuggestions: string[] = [];
|
|
|
|
const desc = scoreDescription(skill.frontmatter, skill.body);
|
|
const fm = scoreFrontmatter(skill.frontmatter);
|
|
const conc = scoreConciseness(skill.body);
|
|
const act = scoreActionability(skill.body);
|
|
const str = scoreStructure(skill.body);
|
|
const trg = scoreTriggers(skill.frontmatter, skill.body);
|
|
const code = scoreCodeExamples(skill.body);
|
|
const meta = scoreMetadata(skill.frontmatter);
|
|
|
|
const dimensions: DimensionScores = {
|
|
description: desc.score,
|
|
frontmatter: fm.score,
|
|
conciseness: conc.score,
|
|
actionability: act.score,
|
|
structure: str.score,
|
|
triggers: trg.score,
|
|
codeExamples: code.score,
|
|
metadata: meta.score,
|
|
};
|
|
|
|
// Weighted score out of 100
|
|
let weighted = 0;
|
|
for (const [key, weight] of Object.entries(WEIGHTS)) {
|
|
weighted += (dimensions[key as keyof DimensionScores] / 10) * weight;
|
|
}
|
|
const score = Math.round(weighted);
|
|
|
|
allSuggestions.push(...desc.suggestions, ...fm.suggestions, ...conc.suggestions,
|
|
...act.suggestions, ...str.suggestions, ...trg.suggestions, ...code.suggestions, ...meta.suggestions);
|
|
|
|
const grade = score >= 85 ? 'A' : score >= 70 ? 'B' : score >= 55 ? 'C' : score >= 40 ? 'D' : 'F';
|
|
|
|
return { name: skill.name, score, grade, dimensions, suggestions: allSuggestions };
|
|
}
|
|
|
|
// ============================================================================
|
|
// OUTPUT
|
|
// ============================================================================
|
|
|
|
function formatResults(scores: SkillScore[], showSuggestions: boolean): string {
|
|
const sep = '='.repeat(120);
|
|
const dimHeaders = 'Desc FM Conc Act Str Trg Code Meta';
|
|
const header = `Skill${' '.repeat(39)}Score Grade ${dimHeaders}`;
|
|
const divider = '-'.repeat(120);
|
|
|
|
let out = `\n${sep}\nSKILL QUALITY SCORER\n${scores.length} AQE skills analyzed\n${sep}\n${header}\n${divider}\n`;
|
|
|
|
for (const s of scores) {
|
|
const d = s.dimensions;
|
|
const name = s.name.length > 42 ? s.name.slice(0, 39) + '...' : s.name.padEnd(42);
|
|
out += `${name} ${String(s.score).padStart(3)} ${s.grade} ${String(d.description).padStart(2)} ${String(d.frontmatter).padStart(2)} ${String(d.conciseness).padStart(2)} ${String(d.actionability).padStart(2)} ${String(d.structure).padStart(2)} ${String(d.triggers).padStart(2)} ${String(d.codeExamples).padStart(2)} ${String(d.metadata).padStart(2)}\n`;
|
|
}
|
|
|
|
// Summary
|
|
const grades = { A: 0, B: 0, C: 0, D: 0, F: 0 };
|
|
for (const s of scores) grades[s.grade as keyof typeof grades]++;
|
|
const avg = Math.round(scores.reduce((sum, s) => sum + s.score, 0) / scores.length);
|
|
const sorted = [...scores].sort((a, b) => a.score - b.score);
|
|
const median = sorted.length % 2 === 0
|
|
? Math.round((sorted[sorted.length / 2 - 1].score + sorted[sorted.length / 2].score) / 2)
|
|
: sorted[Math.floor(sorted.length / 2)].score;
|
|
|
|
out += `${sep}\nSUMMARY: A=${grades.A} B=${grades.B} C=${grades.C} D=${grades.D} F=${grades.F} | avg=${avg} median=${median}\n${sep}\n`;
|
|
|
|
if (showSuggestions) {
|
|
out += '\nIMPROVEMENT SUGGESTIONS:\n';
|
|
for (const s of scores) {
|
|
if (s.suggestions.length === 0) continue;
|
|
out += `\n ${s.name} (${s.grade}, ${s.score}):\n`;
|
|
for (const sug of s.suggestions) {
|
|
out += ` - ${sug}\n`;
|
|
}
|
|
}
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
// ============================================================================
|
|
// MAIN
|
|
// ============================================================================
|
|
|
|
function main() {
|
|
const args = process.argv.slice(2);
|
|
let filterSkill: string | undefined;
|
|
let showSuggestions = false;
|
|
let jsonOutput = false;
|
|
let minScore: number | undefined;
|
|
|
|
for (let i = 0; i < args.length; i++) {
|
|
switch (args[i]) {
|
|
case '--skill': filterSkill = args[++i]; break;
|
|
case '--suggestions': showSuggestions = true; break;
|
|
case '--json': jsonOutput = true; break;
|
|
case '--min-score': minScore = parseInt(args[++i], 10); break;
|
|
case '--help': case '-h':
|
|
console.log(`
|
|
Skill Quality Scorer
|
|
|
|
Usage:
|
|
npx tsx scripts/score-skill-quality.ts [options]
|
|
|
|
Options:
|
|
--skill <name> Score a single skill
|
|
--suggestions Show improvement suggestions
|
|
--json Write results to scripts/skill-quality-scores.json
|
|
--min-score <n> Only show skills scoring >= n
|
|
--help Show this help
|
|
`);
|
|
process.exit(0);
|
|
}
|
|
}
|
|
|
|
const projectRoot = getProjectRoot();
|
|
const skills = discoverSkills(projectRoot, filterSkill);
|
|
|
|
if (skills.length === 0) {
|
|
console.error(filterSkill ? `Skill not found: ${filterSkill}` : 'No skills found');
|
|
process.exit(1);
|
|
}
|
|
|
|
let scores = skills.map(scoreSkill);
|
|
scores.sort((a, b) => b.score - a.score);
|
|
|
|
if (minScore !== undefined) {
|
|
scores = scores.filter(s => s.score >= minScore);
|
|
}
|
|
|
|
console.log(formatResults(scores, showSuggestions));
|
|
|
|
if (jsonOutput) {
|
|
const jsonPath = join(projectRoot, 'scripts', 'skill-quality-scores.json');
|
|
const data = {
|
|
generatedAt: new Date().toISOString(),
|
|
skillCount: scores.length,
|
|
scores: scores.map(s => ({
|
|
name: s.name,
|
|
score: s.score,
|
|
grade: s.grade,
|
|
dimensions: s.dimensions,
|
|
suggestions: s.suggestions,
|
|
})),
|
|
};
|
|
writeFileSync(jsonPath, JSON.stringify(data, null, 2));
|
|
console.log(`JSON results written to ${jsonPath}`);
|
|
}
|
|
}
|
|
|
|
main();
|