mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Profa
fd2f37eaa0
Changes: - Fix README.md agent counts (48 QE agents = 41 main + 7 subagents) - Fix README.md skill counts (15 QE-specific skills) - Fix v3/README.md agent counts (47 → 48) - Fix init-wizard.ts hooks from 'npx @agentic-qe/v3' to 'npx agentic-qe' - Complete v2 → v2/ folder reorganization (54 items moved) - Update root package.json to point to v3 as main version Verified claims: - 48 QE agents (41 main + 7 TDD subagents) - 15 QE-specific skills - 12 DDD bounded contexts - ~5,600+ tests - HNSW, ReasoningBank, Queen Coordinator features exist Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
642 lines
22 KiB
TypeScript
642 lines
22 KiB
TypeScript
#!/usr/bin/env tsx
|
||
|
||
/**
|
||
* Verify Features Script
|
||
*
|
||
* Comprehensive verification of feature claims in README.md against actual implementation.
|
||
* Checks for existence of classes, tests, and validates performance claims.
|
||
*
|
||
* Usage:
|
||
* npm run verify:features
|
||
* tsx scripts/verify-features.ts
|
||
* tsx scripts/verify-features.ts --verbose
|
||
* tsx scripts/verify-features.ts --json
|
||
* tsx scripts/verify-features.ts --feature multi-model-router
|
||
*
|
||
* Exit codes:
|
||
* 0 - All features verified
|
||
* 1 - Features missing or claims unverified
|
||
*
|
||
* @author Agentic QE Team
|
||
* @version 1.0.0
|
||
*/
|
||
|
||
import * as fs from 'fs';
|
||
import * as path from 'path';
|
||
import { execSync } from 'child_process';
|
||
|
||
interface FeatureCheck {
|
||
name: string;
|
||
type: 'class' | 'file' | 'config' | 'test' | 'performance' | 'integration';
|
||
description: string;
|
||
status: 'pass' | 'fail' | 'warning' | 'skip';
|
||
details?: string;
|
||
path?: string;
|
||
}
|
||
|
||
interface FeatureVerification {
|
||
feature: string;
|
||
description: string;
|
||
claimed: string;
|
||
checks: FeatureCheck[];
|
||
overallStatus: 'verified' | 'partial' | 'missing' | 'unknown';
|
||
confidence: number; // 0-100
|
||
}
|
||
|
||
interface VerificationReport {
|
||
timestamp: string;
|
||
summary: {
|
||
totalFeatures: number;
|
||
verified: number;
|
||
partial: number;
|
||
missing: number;
|
||
unknown: number;
|
||
averageConfidence: number;
|
||
};
|
||
features: FeatureVerification[];
|
||
errors: string[];
|
||
}
|
||
|
||
const VERBOSE = process.argv.includes('--verbose');
|
||
const JSON_OUTPUT = process.argv.includes('--json');
|
||
const SPECIFIC_FEATURE = process.argv.find(arg => arg.startsWith('--feature='))?.split('=')[1];
|
||
const PROJECT_ROOT = path.resolve(__dirname, '..');
|
||
|
||
/**
|
||
* Check if a file exists
|
||
*/
|
||
function checkFileExists(relativePath: string): FeatureCheck {
|
||
const fullPath = path.join(PROJECT_ROOT, relativePath);
|
||
const exists = fs.existsSync(fullPath);
|
||
|
||
return {
|
||
name: `File: ${relativePath}`,
|
||
type: 'file',
|
||
description: `Check if ${relativePath} exists`,
|
||
status: exists ? 'pass' : 'fail',
|
||
path: exists ? fullPath : undefined,
|
||
details: exists ? `Found at ${fullPath}` : `Not found at ${fullPath}`
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Check if a class exists in a file
|
||
*/
|
||
function checkClassExists(file: string, className: string): FeatureCheck {
|
||
const fullPath = path.join(PROJECT_ROOT, file);
|
||
|
||
if (!fs.existsSync(fullPath)) {
|
||
return {
|
||
name: `Class: ${className}`,
|
||
type: 'class',
|
||
description: `Check if class ${className} exists in ${file}`,
|
||
status: 'fail',
|
||
details: `File not found: ${fullPath}`
|
||
};
|
||
}
|
||
|
||
const content = fs.readFileSync(fullPath, 'utf-8');
|
||
const classPattern = new RegExp(`(class|interface|type)\\s+${className}\\b`);
|
||
const exists = classPattern.test(content);
|
||
|
||
return {
|
||
name: `Class: ${className}`,
|
||
type: 'class',
|
||
description: `Check if class ${className} exists in ${file}`,
|
||
status: exists ? 'pass' : 'fail',
|
||
path: exists ? fullPath : undefined,
|
||
details: exists ? `Found in ${file}` : `Not found in ${file}`
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Check if tests exist for a feature (supports recursive patterns)
|
||
*/
|
||
function checkTestsExist(pattern: string, keywords?: string[]): FeatureCheck {
|
||
const displayPattern = keywords ? keywords.join(', ') : pattern;
|
||
|
||
try {
|
||
let files: string[] = [];
|
||
|
||
// If keywords provided, use grep for flexible matching on filenames
|
||
if (keywords && keywords.length > 0) {
|
||
const keywordPattern = keywords.join('|');
|
||
// Match case-insensitively on file paths
|
||
const result = execSync(
|
||
`find ${PROJECT_ROOT}/tests -type f -name "*.test.ts" | grep -iE "${keywordPattern}" || true`,
|
||
{
|
||
encoding: 'utf-8'
|
||
}
|
||
);
|
||
files = result.trim().split('\n').filter(f => f.length > 0);
|
||
} else {
|
||
// Use find with pattern
|
||
const result = execSync(
|
||
`find ${PROJECT_ROOT}/tests -type f -name "${pattern}" || true`,
|
||
{
|
||
encoding: 'utf-8'
|
||
}
|
||
);
|
||
files = result.trim().split('\n').filter(f => f.length > 0);
|
||
}
|
||
|
||
const exists = files.length > 0;
|
||
|
||
return {
|
||
name: `Tests: ${displayPattern}`,
|
||
type: 'test',
|
||
description: `Check if tests matching ${displayPattern} exist`,
|
||
status: exists ? 'pass' : 'warning',
|
||
details: exists
|
||
? `Found ${files.length} test file(s)${VERBOSE ? ':\n ' + files.map(f => path.relative(PROJECT_ROOT, f)).join('\n ') : ''}`
|
||
: `No tests found matching ${displayPattern}`
|
||
};
|
||
} catch (error) {
|
||
return {
|
||
name: `Tests: ${displayPattern}`,
|
||
type: 'test',
|
||
description: `Check if tests matching ${displayPattern} exist`,
|
||
status: 'warning',
|
||
details: `Error scanning for tests: ${error instanceof Error ? error.message : String(error)}`
|
||
};
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Check if configuration exists
|
||
*/
|
||
function checkConfigExists(configPath: string, key?: string): FeatureCheck {
|
||
const fullPath = path.join(PROJECT_ROOT, configPath);
|
||
|
||
if (!fs.existsSync(fullPath)) {
|
||
return {
|
||
name: `Config: ${configPath}`,
|
||
type: 'config',
|
||
description: `Check if configuration ${configPath} exists`,
|
||
status: 'fail',
|
||
details: `Configuration file not found`
|
||
};
|
||
}
|
||
|
||
if (!key) {
|
||
return {
|
||
name: `Config: ${configPath}`,
|
||
type: 'config',
|
||
description: `Check if configuration ${configPath} exists`,
|
||
status: 'pass',
|
||
path: fullPath,
|
||
details: `Configuration file found`
|
||
};
|
||
}
|
||
|
||
try {
|
||
const content = fs.readFileSync(fullPath, 'utf-8');
|
||
const config = JSON.parse(content);
|
||
const hasKey = key.split('.').reduce((obj, k) => obj?.[k], config) !== undefined;
|
||
|
||
return {
|
||
name: `Config: ${configPath} (${key})`,
|
||
type: 'config',
|
||
description: `Check if configuration key ${key} exists`,
|
||
status: hasKey ? 'pass' : 'fail',
|
||
path: fullPath,
|
||
details: hasKey ? `Key ${key} found in config` : `Key ${key} not found`
|
||
};
|
||
} catch (error) {
|
||
return {
|
||
name: `Config: ${configPath}`,
|
||
type: 'config',
|
||
description: `Check if configuration ${configPath} is valid`,
|
||
status: 'fail',
|
||
details: `Invalid JSON: ${error}`
|
||
};
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Check if code pattern exists
|
||
*/
|
||
function checkCodePattern(file: string, pattern: RegExp, description: string): FeatureCheck {
|
||
const fullPath = path.join(PROJECT_ROOT, file);
|
||
|
||
if (!fs.existsSync(fullPath)) {
|
||
return {
|
||
name: description,
|
||
type: 'file',
|
||
description,
|
||
status: 'fail',
|
||
details: `File not found: ${fullPath}`
|
||
};
|
||
}
|
||
|
||
const content = fs.readFileSync(fullPath, 'utf-8');
|
||
const matches = content.match(pattern);
|
||
const exists = !!matches;
|
||
|
||
return {
|
||
name: description,
|
||
type: 'file',
|
||
description,
|
||
status: exists ? 'pass' : 'fail',
|
||
path: exists ? fullPath : undefined,
|
||
details: exists ? `Pattern found (${matches?.length || 0} matches)` : `Pattern not found`
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Verify Multi-Model Router feature
|
||
*/
|
||
function verifyMultiModelRouter(): FeatureVerification {
|
||
const checks: FeatureCheck[] = [
|
||
checkClassExists('src/core/routing/AdaptiveModelRouter.ts', 'AdaptiveModelRouter'),
|
||
checkConfigExists('.agentic-qe/config/routing.json'),
|
||
checkConfigExists('.agentic-qe/config/routing.json', 'multiModelRouter'),
|
||
checkClassExists('src/core/routing/CostTracker.ts', 'CostTracker'),
|
||
checkTestsExist('*routing*.test.ts', ['routing', 'router']),
|
||
checkTestsExist('*model*.test.ts', ['AdaptiveModelRouter', 'ModelRouter', 'CostTracker']),
|
||
checkCodePattern('src/core/routing/AdaptiveModelRouter.ts', /selectModel|chooseModel/, 'Model selection logic'),
|
||
checkCodePattern('src/core/routing/CostTracker.ts', /trackCost|calculateCost/, 'Cost tracking implementation')
|
||
];
|
||
|
||
const passCount = checks.filter(c => c.status === 'pass').length;
|
||
const confidence = (passCount / checks.length) * 100;
|
||
|
||
return {
|
||
feature: 'Multi-Model Router',
|
||
description: '70-81% cost savings through intelligent model selection',
|
||
claimed: '70-81% cost savings, intelligent model selection based on task complexity',
|
||
checks,
|
||
overallStatus: confidence >= 80 ? 'verified' : confidence >= 50 ? 'partial' : 'missing',
|
||
confidence
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Verify Learning System feature
|
||
*/
|
||
function verifyLearningSystem(): FeatureVerification {
|
||
const checks: FeatureCheck[] = [
|
||
checkClassExists('src/learning/LearningEngine.ts', 'LearningEngine'),
|
||
checkClassExists('src/learning/QLearning.ts', 'QLearning'),
|
||
checkFileExists('src/learning/ExperienceReplayBuffer.ts'),
|
||
checkTestsExist('*learning*.test.ts', ['learning', 'LearningEngine', 'ImprovementLoop']),
|
||
checkTestsExist('*qlearning*.test.ts', ['QLearning', 'qlearning']),
|
||
checkCodePattern('src/learning/QLearning.ts', /updateQValue|qTable/, 'Q-learning algorithm'),
|
||
checkCodePattern('src/learning/LearningEngine.ts', /train|learn/, 'Learning implementation'),
|
||
checkFileExists('.agentic-qe/data/learning/state.json')
|
||
];
|
||
|
||
const passCount = checks.filter(c => c.status === 'pass').length;
|
||
const confidence = (passCount / checks.length) * 100;
|
||
|
||
return {
|
||
feature: 'Learning System',
|
||
description: '20% continuous improvement through Q-learning',
|
||
claimed: '20% improvement target, Q-learning algorithm, experience replay',
|
||
checks,
|
||
overallStatus: confidence >= 80 ? 'verified' : confidence >= 50 ? 'partial' : 'missing',
|
||
confidence
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Verify Pattern Bank feature
|
||
*/
|
||
function verifyPatternBank(): FeatureVerification {
|
||
const checks: FeatureCheck[] = [
|
||
checkClassExists('src/reasoning/QEReasoningBank.ts', 'QEReasoningBank'),
|
||
checkCodePattern('src/reasoning/QEReasoningBank.ts', /extractPattern|storePattern/, 'Pattern extraction'),
|
||
checkCodePattern('src/reasoning/QEReasoningBank.ts', /matchPattern|findPattern/, 'Pattern matching'),
|
||
checkTestsExist('*pattern*.test.ts', ['Pattern', 'PatternExtractor', 'PatternClassifier']),
|
||
checkTestsExist('*reasoning*.test.ts', ['Reasoning', 'QEReasoningBank', 'VectorSimilarity']),
|
||
checkFileExists('.agentic-qe/data/patterns/registry.json'),
|
||
checkCodePattern('src/reasoning/QEReasoningBank.ts', /share|sync/, 'Cross-project sharing')
|
||
];
|
||
|
||
const passCount = checks.filter(c => c.status === 'pass').length;
|
||
const confidence = (passCount / checks.length) * 100;
|
||
|
||
return {
|
||
feature: 'Pattern Bank (QE Reasoning Bank)',
|
||
description: '85%+ matching accuracy for pattern reuse',
|
||
claimed: '85%+ matching accuracy, pattern extraction, cross-project sharing',
|
||
checks,
|
||
overallStatus: confidence >= 80 ? 'verified' : confidence >= 50 ? 'partial' : 'missing',
|
||
confidence
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Verify ML Flaky Detection feature
|
||
*/
|
||
function verifyFlakyDetection(): FeatureVerification {
|
||
const checks: FeatureCheck[] = [
|
||
checkClassExists('src/learning/FlakyTestDetector.ts', 'FlakyTestDetector'),
|
||
checkCodePattern('src/learning/FlakyTestDetector.ts', /detectFlaky|isFlaky/, 'Flaky detection logic'),
|
||
checkCodePattern('src/learning/FlakyTestDetector.ts', /rootCause|analyze/, 'Root cause analysis'),
|
||
checkCodePattern('src/learning/FlakyTestDetector.ts', /recommend|fix/, 'Fix recommendations'),
|
||
checkTestsExist('*flaky*.test.ts', ['Flaky', 'FlakyTestDetector']),
|
||
checkCodePattern('src/learning/FlakyTestDetector.ts', /ml|model|predict/, 'ML model usage')
|
||
];
|
||
|
||
const passCount = checks.filter(c => c.status === 'pass').length;
|
||
const confidence = (passCount / checks.length) * 100;
|
||
|
||
return {
|
||
feature: 'ML Flaky Test Detection',
|
||
description: '100% accuracy in flaky test detection',
|
||
claimed: '100% accuracy, ML-based detection, root cause analysis, fix recommendations',
|
||
checks,
|
||
overallStatus: confidence >= 80 ? 'verified' : confidence >= 50 ? 'partial' : 'missing',
|
||
confidence
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Verify Streaming API feature
|
||
*/
|
||
function verifyStreamingAPI(): FeatureVerification {
|
||
const checks: FeatureCheck[] = [
|
||
checkFileExists('src/mcp/streaming/TestExecuteStreamHandler.ts'),
|
||
checkFileExists('src/mcp/streaming/CoverageAnalyzeStreamHandler.ts'),
|
||
checkCodePattern('src/mcp/streaming/TestExecuteStreamHandler.ts', /AsyncGenerator|async\s*\*/, 'AsyncGenerator pattern'),
|
||
checkCodePattern('src/mcp/streaming/TestExecuteStreamHandler.ts', /progress|emit/, 'Progress events'),
|
||
checkTestsExist('*stream*.test.ts', ['stream', 'Stream', 'streaming']),
|
||
checkCodePattern('src/mcp/tools.ts', /test_execute_stream|coverage_analyze_stream/, 'Streaming tool definitions')
|
||
];
|
||
|
||
const passCount = checks.filter(c => c.status === 'pass').length;
|
||
const confidence = (passCount / checks.length) * 100;
|
||
|
||
return {
|
||
feature: 'Streaming API',
|
||
description: 'Real-time progress updates for long-running operations',
|
||
claimed: 'AsyncGenerator pattern, real-time progress, for-await-of compatibility',
|
||
checks,
|
||
overallStatus: confidence >= 80 ? 'verified' : confidence >= 50 ? 'partial' : 'missing',
|
||
confidence
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Verify AgentDB Integration feature
|
||
*/
|
||
function verifyAgentDBIntegration(): FeatureVerification {
|
||
const checks: FeatureCheck[] = [
|
||
checkFileExists('package.json'),
|
||
checkCodePattern('package.json', /"agentdb[^"]*":\s*"[^"]*"/, 'AgentDB dependency'),
|
||
checkCodePattern('src/agents/BaseAgent.ts', /agentdb|AgentDB/i, 'AgentDB imports'),
|
||
checkCodePattern('src/agents/BaseAgent.ts', /quic|QUIC/i, 'QUIC sync usage'),
|
||
checkCodePattern('src/agents/BaseAgent.ts', /vectorSearch|vector/, 'Vector search usage'),
|
||
checkTestsExist('*agentdb*.test.ts', ['agentdb', 'AgentDB'])
|
||
];
|
||
|
||
const passCount = checks.filter(c => c.status === 'pass').length;
|
||
const confidence = (passCount / checks.length) * 100;
|
||
|
||
return {
|
||
feature: 'AgentDB Integration',
|
||
description: 'Vector database with QUIC sync and learning plugins',
|
||
claimed: 'AgentDB installed, QUIC sync, vector search, learning plugins, performance optimizations',
|
||
checks,
|
||
overallStatus: confidence >= 80 ? 'verified' : confidence >= 50 ? 'partial' : 'missing',
|
||
confidence
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Verify MCP Tools count
|
||
*/
|
||
function verifyMCPTools(): FeatureVerification {
|
||
const checks: FeatureCheck[] = [
|
||
checkFileExists('src/mcp/tools.ts'),
|
||
checkCodePattern('src/mcp/tools.ts', /export\s+const\s+agenticQETools/, 'Tools export'),
|
||
checkCodePattern('src/mcp/tools.ts', /mcp__agentic_qe__/g, 'Tool definitions'),
|
||
checkFileExists('src/mcp/handlers'),
|
||
checkTestsExist('*mcp*.test.ts', ['mcp', 'MCP']),
|
||
checkFileExists('bin/aqe-mcp')
|
||
];
|
||
|
||
const passCount = checks.filter(c => c.status === 'pass').length;
|
||
const confidence = (passCount / checks.length) * 100;
|
||
|
||
// Count actual tools
|
||
const toolsFile = path.join(PROJECT_ROOT, 'src/mcp/tools.ts');
|
||
let toolCount = 0;
|
||
if (fs.existsSync(toolsFile)) {
|
||
const content = fs.readFileSync(toolsFile, 'utf-8');
|
||
const matches = content.match(/name:\s*['"]mcp__agentic_qe__[^'"]+['"]/g);
|
||
toolCount = matches ? matches.length : 0;
|
||
}
|
||
|
||
return {
|
||
feature: 'MCP Tools',
|
||
description: `${toolCount} MCP tools for Claude Code integration`,
|
||
claimed: '61 MCP tools for comprehensive QE operations',
|
||
checks,
|
||
overallStatus: confidence >= 80 ? 'verified' : confidence >= 50 ? 'partial' : 'missing',
|
||
confidence
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Verify performance claims
|
||
*/
|
||
function verifyPerformanceClaims(): FeatureVerification {
|
||
const checks: FeatureCheck[] = [
|
||
checkTestsExist('*performance*.test.ts', ['performance', 'Performance', 'benchmark']),
|
||
checkTestsExist('*benchmark*.test.ts', ['benchmark', 'Benchmark']),
|
||
checkCodePattern('src/test-generation/TestGenerator.ts', /1000|rate|throughput/, 'Test generation performance'),
|
||
checkCodePattern('src/coverage/CoverageAnalyzer.ts', /O\(log\s*n\)|sublinear|johnson/, 'O(log n) complexity'),
|
||
checkCodePattern('src/test-data/TestDataGenerator.ts', /10000|10,000/, 'Data generation rate'),
|
||
checkTestsExist('*sublinear*.test.ts', ['sublinear', 'Sublinear']),
|
||
checkFileExists('tests/performance')
|
||
];
|
||
|
||
const passCount = checks.filter(c => c.status === 'pass').length;
|
||
const confidence = (passCount / checks.length) * 100;
|
||
|
||
return {
|
||
feature: 'Performance Claims',
|
||
description: 'High-performance test generation and analysis',
|
||
claimed: '1000+ tests/min, O(log n) coverage, 10,000+ records/sec, <50ms p95 matching',
|
||
checks,
|
||
overallStatus: confidence >= 80 ? 'verified' : confidence >= 50 ? 'partial' : 'missing',
|
||
confidence
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Run comprehensive verification
|
||
*/
|
||
function verify(): VerificationReport {
|
||
const report: VerificationReport = {
|
||
timestamp: new Date().toISOString(),
|
||
summary: {
|
||
totalFeatures: 0,
|
||
verified: 0,
|
||
partial: 0,
|
||
missing: 0,
|
||
unknown: 0,
|
||
averageConfidence: 0
|
||
},
|
||
features: [],
|
||
errors: []
|
||
};
|
||
|
||
try {
|
||
const features = [
|
||
verifyMultiModelRouter,
|
||
verifyLearningSystem,
|
||
verifyPatternBank,
|
||
verifyFlakyDetection,
|
||
verifyStreamingAPI,
|
||
verifyAgentDBIntegration,
|
||
verifyMCPTools,
|
||
verifyPerformanceClaims
|
||
];
|
||
|
||
for (const verifyFeature of features) {
|
||
try {
|
||
const result = verifyFeature();
|
||
|
||
// Skip if specific feature requested and this isn't it
|
||
if (SPECIFIC_FEATURE && !result.feature.toLowerCase().includes(SPECIFIC_FEATURE.toLowerCase())) {
|
||
continue;
|
||
}
|
||
|
||
report.features.push(result);
|
||
} catch (error) {
|
||
report.errors.push(`Error verifying feature: ${error}`);
|
||
}
|
||
}
|
||
|
||
// Calculate summary
|
||
report.summary.totalFeatures = report.features.length;
|
||
report.summary.verified = report.features.filter(f => f.overallStatus === 'verified').length;
|
||
report.summary.partial = report.features.filter(f => f.overallStatus === 'partial').length;
|
||
report.summary.missing = report.features.filter(f => f.overallStatus === 'missing').length;
|
||
report.summary.unknown = report.features.filter(f => f.overallStatus === 'unknown').length;
|
||
|
||
const totalConfidence = report.features.reduce((sum, f) => sum + f.confidence, 0);
|
||
report.summary.averageConfidence = report.features.length > 0 ? totalConfidence / report.features.length : 0;
|
||
|
||
} catch (error) {
|
||
report.errors.push(`Verification error: ${error}`);
|
||
}
|
||
|
||
return report;
|
||
}
|
||
|
||
/**
|
||
* Display report
|
||
*/
|
||
function displayReport(report: VerificationReport): void {
|
||
if (JSON_OUTPUT) {
|
||
console.log(JSON.stringify(report, null, 2));
|
||
return;
|
||
}
|
||
|
||
console.log('\n' + '='.repeat(80));
|
||
console.log('FEATURE VERIFICATION REPORT');
|
||
console.log('='.repeat(80));
|
||
console.log(`Generated: ${report.timestamp}\n`);
|
||
|
||
for (const feature of report.features) {
|
||
const statusIcon = {
|
||
verified: '✅',
|
||
partial: '⚠️',
|
||
missing: '❌',
|
||
unknown: '❓'
|
||
}[feature.overallStatus];
|
||
|
||
console.log(`\n${statusIcon} ${feature.feature}`);
|
||
console.log('-'.repeat(80));
|
||
console.log(`Claimed: ${feature.claimed}`);
|
||
console.log(`Status: ${feature.overallStatus.toUpperCase()} (${feature.confidence.toFixed(1)}% confidence)\n`);
|
||
|
||
const passChecks = feature.checks.filter(c => c.status === 'pass');
|
||
const failChecks = feature.checks.filter(c => c.status === 'fail');
|
||
const warnChecks = feature.checks.filter(c => c.status === 'warning');
|
||
|
||
console.log(`Checks: ${passChecks.length} passed, ${failChecks.length} failed, ${warnChecks.length} warnings`);
|
||
|
||
if (VERBOSE || feature.overallStatus !== 'verified') {
|
||
console.log('\nDetailed Checks:');
|
||
for (const check of feature.checks) {
|
||
const checkIcon = check.status === 'pass' ? '✓' : check.status === 'fail' ? '✗' : '⚠';
|
||
console.log(` ${checkIcon} ${check.name}`);
|
||
if (VERBOSE || check.status !== 'pass') {
|
||
console.log(` ${check.details}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (feature.overallStatus === 'partial' || feature.overallStatus === 'missing') {
|
||
console.log('\n💡 Action Required:');
|
||
for (const check of failChecks) {
|
||
console.log(` • ${check.description}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log('\n' + '='.repeat(80));
|
||
console.log('SUMMARY');
|
||
console.log('='.repeat(80));
|
||
console.log(`Total Features: ${report.summary.totalFeatures}`);
|
||
console.log(`✅ Verified: ${report.summary.verified}`);
|
||
console.log(`⚠️ Partial: ${report.summary.partial}`);
|
||
console.log(`❌ Missing: ${report.summary.missing}`);
|
||
console.log(`❓ Unknown: ${report.summary.unknown}`);
|
||
console.log(`\nAverage Confidence: ${report.summary.averageConfidence.toFixed(1)}%`);
|
||
|
||
if (report.errors.length > 0) {
|
||
console.log('\n⚠️ ERRORS:');
|
||
report.errors.forEach(error => console.log(` - ${error}`));
|
||
}
|
||
|
||
console.log('='.repeat(80) + '\n');
|
||
|
||
// Overall assessment
|
||
const overallConfidence = report.summary.averageConfidence;
|
||
if (overallConfidence >= 80) {
|
||
console.log('🎉 EXCELLENT: High confidence in documentation claims\n');
|
||
} else if (overallConfidence >= 60) {
|
||
console.log('👍 GOOD: Most features verified, some improvements needed\n');
|
||
} else {
|
||
console.log('⚠️ NEEDS WORK: Significant gaps between claims and implementation\n');
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Main execution
|
||
*/
|
||
function main(): void {
|
||
const report = verify();
|
||
displayReport(report);
|
||
|
||
// Save report
|
||
const reportsDir = path.join(PROJECT_ROOT, 'reports');
|
||
if (!fs.existsSync(reportsDir)) {
|
||
fs.mkdirSync(reportsDir, { recursive: true });
|
||
}
|
||
|
||
const reportPath = path.join(reportsDir, `verification-features-${Date.now()}.json`);
|
||
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
|
||
|
||
if (VERBOSE) {
|
||
console.log(`📄 Full report saved to: ${path.relative(PROJECT_ROOT, reportPath)}\n`);
|
||
}
|
||
|
||
// Exit with appropriate code
|
||
const hasIssues = report.summary.missing > 0 || report.summary.averageConfidence < 80;
|
||
process.exit(hasIssues ? 1 : 0);
|
||
}
|
||
|
||
// Run if executed directly
|
||
if (require.main === module) {
|
||
main();
|
||
}
|
||
|
||
export { verify, VerificationReport, FeatureVerification };
|