proffesor-for-testing__agentic-qe/scripts/validation/phase2-vc3-clause-evaluation.ts

#!/usr/bin/env ts-node
/**
 * Phase 2 Validation Criterion 3: Clause Evaluation
 *
 * Tests: Can we evaluate a clause against a file?
 * Expected: Evaluators work and return verdict <5s
 *
 * Plan requirement:
 * | Clause Evaluation | `aqe constitution evaluate --clause C001 test.ts` | Verdict with findings |
 */

import { ASTEvaluator } from '../../src/constitution/evaluators/ast-evaluator';
import { MetricEvaluator } from '../../src/constitution/evaluators/metric-evaluator';
import { PatternEvaluator } from '../../src/constitution/evaluators/pattern-evaluator';
import { SemanticEvaluator } from '../../src/constitution/evaluators/semantic-evaluator';
import { CheckType, EvaluationContext } from '../../src/constitution/evaluators/base';
import { RuleCondition } from '../../src/constitution/schema';

// Sample TypeScript code for evaluation
const SAMPLE_CODE = `
export class UserService {
  private users: Map<string, User> = new Map();

  async createUser(data: UserData): Promise<User> {
    if (!data.email || !data.name) {
      throw new Error('Invalid user data');
    }

    const user: User = {
      id: generateId(),
      email: data.email,
      name: data.name,
      createdAt: new Date()
    };

    this.users.set(user.id, user);
    return user;
  }

  async getUser(id: string): Promise<User | undefined> {
    return this.users.get(id);
  }

  async deleteUser(id: string): Promise<boolean> {
    return this.users.delete(id);
  }
}
`;

async function validateVC3(): Promise<void> {
  console.log('='.repeat(60));
  console.log('Phase 2 VC3: Clause Evaluation');
  console.log('='.repeat(60));

  try {
    const startTime = Date.now();

    // Create evaluation context
    const context: EvaluationContext = {
      filePath: 'test/UserService.ts',
      sourceCode: SAMPLE_CODE,
      language: 'typescript',
      data: {
        project: 'validation-test',
        timestamp: new Date().toISOString()
      }
    };

    // Test 1: AST Evaluator
    console.log('\n[Test 1] AST Evaluator - Complexity Check...');
    const astEvaluator = new ASTEvaluator();
    const astCondition: RuleCondition = {
      field: 'complexity',
      operator: 'less_than',
      value: 10
    };

    const astResult = await astEvaluator.evaluate(astCondition, context);
    console.log(`✓ AST evaluation completed`);
    console.log(`  Pass: ${astResult.passed}`);
    console.log(`  Actual complexity: ${astResult.actualValue}`);

    // Test 2: Metric Evaluator
    console.log('\n[Test 2] Metric Evaluator - LOC Check...');
    const metricEvaluator = new MetricEvaluator();
    const metricCondition: RuleCondition = {
      field: 'lines_of_code',
      operator: 'less_than',
      value: 500
    };

    const metricResult = await metricEvaluator.evaluate(metricCondition, context);
    console.log(`✓ Metric evaluation completed`);
    console.log(`  Pass: ${metricResult.passed}`);
    console.log(`  Lines of code: ${metricResult.actualValue}`);

    // Test 3: Pattern Evaluator
    console.log('\n[Test 3] Pattern Evaluator - Async/Await Check...');
    const patternEvaluator = new PatternEvaluator();
    const patternCondition: RuleCondition = {
      field: 'async-await',
      operator: 'matches',
      value: 'async\\s+\\w+\\s*\\([^)]*\\)\\s*:\\s*Promise'
    };

    const patternResult = await patternEvaluator.evaluate(patternCondition, context);
    console.log(`✓ Pattern evaluation completed`);
    console.log(`  Pass: ${patternResult.passed}`);
    console.log(`  Pattern found: ${patternResult.actualValue !== null}`);

    // Test 4: Semantic Evaluator (heuristic mode, no LLM)
    console.log('\n[Test 4] Semantic Evaluator - Naming Quality...');
    const semanticEvaluator = new SemanticEvaluator();
    await semanticEvaluator.initialize({
      type: 'semantic',
      enabled: true,
      options: { llmEnabled: false }
    });
    const semanticCondition: RuleCondition = {
      field: 'naming_quality',
      operator: 'greater_than',
      value: 0.7
    };

    const semanticResult = await semanticEvaluator.evaluate(semanticCondition, context);
    console.log(`✓ Semantic evaluation completed`);
    console.log(`  Pass: ${semanticResult.passed}`);
    console.log(`  Naming quality: ${semanticResult.actualValue}`);

    // Test 5: Performance check (<5s requirement)
    const elapsedTime = Date.now() - startTime;
    console.log(`\n[Test 5] Performance Check...`);
    console.log(`✓ Total evaluation time: ${elapsedTime}ms`);
    if (elapsedTime < 5000) {
      console.log(`✓ Under 5-second requirement (${(elapsedTime/1000).toFixed(2)}s)`);
    } else {
      throw new Error(`Evaluation too slow: ${elapsedTime}ms > 5000ms`);
    }

    // Summary
    console.log('\n' + '='.repeat(60));
    console.log('VC3 RESULT: ✅ PASS');
    console.log('='.repeat(60));
    console.log('✓ AST Evaluator: FUNCTIONAL');
    console.log('✓ Metric Evaluator: FUNCTIONAL');
    console.log('✓ Pattern Evaluator: FUNCTIONAL');
    console.log('✓ Semantic Evaluator: FUNCTIONAL');
    console.log(`✓ Performance: ${(elapsedTime/1000).toFixed(2)}s < 5s ✅`);
    console.log('\nEquivalent to: aqe constitution evaluate --clause C001 test.ts');
    console.log('Functionality: OPERATIONAL ✅');

    process.exit(0);
  } catch (error) {
    console.error('\n' + '='.repeat(60));
    console.error('VC3 RESULT: ❌ FAIL');
    console.error('='.repeat(60));
    console.error('Error:', error instanceof Error ? error.message : error);
    console.error('\nPhase 2 validation criterion 3 NOT MET');
    process.exit(1);
  }
}

// Run validation
validateVC3();