proffesor-for-testing__agentic-qe/scripts/regression-validator.ts

#!/usr/bin/env tsx

/**
 * Regression Validation Specialist
 *
 * Monitors test suite stability, tracks metrics over time,
 * and provides GO/NO-GO recommendations for regression safety net.
 */

import { SwarmMemoryManager } from '../src/core/memory/SwarmMemoryManager';
import { EventBus } from '../src/core/EventBus';
import * as path from 'path';
import { execSync } from 'child_process';
import * as fs from 'fs';

interface TestResults {
  passed: number;
  failed: number;
  skipped: number;
  total: number;
  passRate: number;
  duration: number;
  timestamp: number;
}

interface StabilityMetrics {
  passRate: number;
  coverage: number;
  failureCategories: Record<string, number>;
  trend: 'IMPROVING' | 'STABLE' | 'DEGRADING';
  consecutiveRuns: number;
}

interface RegressionValidation {
  timestamp: number;
  agent: string;
  passRate: number;
  coverage: number;
  stability: string;
  recommendation: 'GO' | 'NO-GO';
  regressionSuiteReady: boolean;
  safetyNetScore: number;
  metrics: StabilityMetrics;
}

class RegressionValidator {
  private memoryStore: SwarmMemoryManager;
  private eventBus: EventBus;
  private dbPath: string;
  private runCount: number = 0;
  private results: TestResults[] = [];

  constructor() {
    this.dbPath = path.join(process.cwd(), '.swarm/memory.db');
    this.memoryStore = new SwarmMemoryManager(this.dbPath);
    this.eventBus = EventBus.getInstance();
  }

  async initialize(): Promise<void> {
    await this.memoryStore.initialize();
    console.log('✅ Regression Validator initialized');
    console.log(`📊 Database: ${this.dbPath}`);
  }

  /**
   * Monitor other agents' progress
   */
  async monitorAgentProgress(): Promise<void> {
    console.log('\n🔍 Monitoring agent progress...');

    try {
      const infraFixes = await this.memoryStore.retrieve('tasks/INFRA-FIX-003/status', {
        partition: 'coordination'
      });
      console.log('  📦 Infrastructure fixes:', infraFixes || 'No data');

      const testFixes = await this.memoryStore.retrieve('tasks/TEST-FIX-BATCH-001/status', {
        partition: 'coordination'
      });
      console.log('  🧪 Test fixes:', testFixes || 'No data');

      const coverage = await this.memoryStore.retrieve('aqe/coverage/improvement-progress', {
        partition: 'coordination'
      });
      console.log('  📈 Coverage progress:', coverage || 'No data');

      // Store monitoring checkpoint
      await this.memoryStore.store('aqe/regression/agent-monitoring', {
        timestamp: Date.now(),
        infraFixes: infraFixes !== null,
        testFixes: testFixes !== null,
        coverageTracked: coverage !== null
      }, { partition: 'coordination', ttl: 3600 });

    } catch (error) {
      console.warn('  ⚠️  Some agent data not available yet');
    }
  }

  /**
   * Run incremental regression tests
   */
  async runRegressionTests(): Promise<TestResults> {
    this.runCount++;
    const timestamp = Date.now();
    const logFile = path.join(process.cwd(), '.swarm/logs', `regression-${timestamp}.log`);

    console.log(`\n🧪 Running regression test #${this.runCount}...`);

    try {
      const output = execSync('npm test -- --passWithNoTests 2>&1', {
        cwd: process.cwd(),
        encoding: 'utf-8',
        maxBuffer: 10 * 1024 * 1024,
        timeout: 300000 // 5 minutes
      });

      // Save log
      fs.writeFileSync(logFile, output);

      // Parse results
      const results = this.parseTestOutput(output, timestamp);
      this.results.push(results);

      console.log(`  ✅ Pass rate: ${results.passRate.toFixed(2)}%`);
      console.log(`  📊 Tests: ${results.passed} passed, ${results.failed} failed, ${results.skipped} skipped`);
      console.log(`  ⏱️  Duration: ${(results.duration / 1000).toFixed(2)}s`);

      return results;

    } catch (error: any) {
      // Jest exits with non-zero when tests fail, but output is still useful
      const output = error.stdout || error.stderr || '';
      fs.writeFileSync(logFile, output);

      const results = this.parseTestOutput(output, timestamp);
      this.results.push(results);

      console.log(`  ⚠️  Pass rate: ${results.passRate.toFixed(2)}%`);
      console.log(`  📊 Tests: ${results.passed} passed, ${results.failed} failed, ${results.skipped} skipped`);

      return results;
    }
  }

  /**
   * Parse Jest test output
   */
  private parseTestOutput(output: string, timestamp: number): TestResults {
    const passedMatch = output.match(/(\d+)\s+passed/);
    const failedMatch = output.match(/(\d+)\s+failed/);
    const skippedMatch = output.match(/(\d+)\s+skipped/);
    const durationMatch = output.match(/Time:\s+([\d.]+)\s*s/);

    const passed = passedMatch ? parseInt(passedMatch[1], 10) : 0;
    const failed = failedMatch ? parseInt(failedMatch[1], 10) : 0;
    const skipped = skippedMatch ? parseInt(skippedMatch[1], 10) : 0;
    const total = passed + failed + skipped;
    const duration = durationMatch ? parseFloat(durationMatch[1]) * 1000 : 0;

    const passRate = total > 0 ? (passed / total) * 100 : 0;

    return {
      passed,
      failed,
      skipped,
      total,
      passRate,
      duration,
      timestamp
    };
  }

  /**
   * Track stability metrics over time
   */
  async trackStabilityMetrics(): Promise<StabilityMetrics> {
    console.log('\n📊 Tracking stability metrics...');

    if (this.results.length < 2) {
      console.log('  ⏳ Not enough data for trend analysis');
      return {
        passRate: this.results[0]?.passRate || 0,
        coverage: 0,
        failureCategories: {},
        trend: 'STABLE',
        consecutiveRuns: this.runCount
      };
    }

    // Calculate trend
    const recentResults = this.results.slice(-5);
    const passRates = recentResults.map(r => r.passRate);
    const avgPassRate = passRates.reduce((a, b) => a + b, 0) / passRates.length;

    // Determine trend
    let trend: 'IMPROVING' | 'STABLE' | 'DEGRADING' = 'STABLE';
    if (passRates.length >= 3) {
      const firstHalf = passRates.slice(0, Math.floor(passRates.length / 2));
      const secondHalf = passRates.slice(Math.floor(passRates.length / 2));
      const firstAvg = firstHalf.reduce((a, b) => a + b, 0) / firstHalf.length;
      const secondAvg = secondHalf.reduce((a, b) => a + b, 0) / secondHalf.length;

      if (secondAvg > firstAvg + 5) trend = 'IMPROVING';
      else if (secondAvg < firstAvg - 5) trend = 'DEGRADING';
    }

    // Get coverage (mock for now, would need actual coverage report)
    const coverage = await this.estimateCoverage();

    // Analyze failure categories
    const failureCategories = await this.analyzeFailureCategories();

    const metrics: StabilityMetrics = {
      passRate: avgPassRate,
      coverage,
      failureCategories,
      trend,
      consecutiveRuns: this.runCount
    };

    // Store in memory
    await this.memoryStore.store('aqe/regression/stability-trend', {
      timestamp: Date.now(),
      metrics,
      history: recentResults
    }, { partition: 'coordination', ttl: 86400 });

    await this.memoryStore.storePerformanceMetric({
      metric: 'regression_stability_trend',
      value: avgPassRate,
      unit: 'percentage',
      timestamp: Date.now()
    });

    console.log(`  📈 Average pass rate: ${avgPassRate.toFixed(2)}%`);
    console.log(`  📉 Trend: ${trend}`);
    console.log(`  🎯 Coverage estimate: ${coverage.toFixed(2)}%`);

    return metrics;
  }

  /**
   * Estimate coverage from test output
   */
  private async estimateCoverage(): Promise<number> {
    try {
      // Try to read actual coverage if available
      const coverageFile = path.join(process.cwd(), 'coverage/coverage-summary.json');
      if (fs.existsSync(coverageFile)) {
        const coverage = JSON.parse(fs.readFileSync(coverageFile, 'utf-8'));
        return coverage.total?.lines?.pct || 0;
      }
    } catch (error) {
      // Fallback to estimation
    }

    // Estimate based on test count (rough heuristic)
    const latestResult = this.results[this.results.length - 1];
    if (latestResult && latestResult.total > 0) {
      // Assume each test covers ~2-3% of codebase
      return Math.min(latestResult.total * 2.5, 100);
    }

    return 0;
  }

  /**
   * Analyze failure categories
   */
  private async analyzeFailureCategories(): Promise<Record<string, number>> {
    const categories: Record<string, number> = {
      infrastructure: 0,
      assertion: 0,
      timeout: 0,
      import: 0,
      other: 0
    };

    // Read recent log files
    const logsDir = path.join(process.cwd(), '.swarm/logs');
    const logFiles = fs.readdirSync(logsDir)
      .filter(f => f.startsWith('regression-'))
      .sort()
      .slice(-5);

    for (const logFile of logFiles) {
      const content = fs.readFileSync(path.join(logsDir, logFile), 'utf-8');

      if (content.includes('Cannot find module')) categories.import++;
      if (content.includes('Timeout')) categories.timeout++;
      if (content.includes('Expected')) categories.assertion++;
      if (content.includes('ECONNREFUSED') || content.includes('ENOENT')) categories.infrastructure++;
    }

    return categories;
  }

  /**
   * Validate regression suite readiness
   */
  async validateRegressionSuite(): Promise<RegressionValidation> {
    console.log('\n✅ Validating regression suite readiness...');

    const latestResult = this.results[this.results.length - 1];
    const metrics = await this.trackStabilityMetrics();

    // Thresholds
    const PASS_RATE_THRESHOLD = 70;
    const COVERAGE_THRESHOLD = 15;
    const MAX_DURATION = 5 * 60 * 1000; // 5 minutes

    const checks = {
      passRate: latestResult.passRate >= PASS_RATE_THRESHOLD,
      coverage: metrics.coverage >= COVERAGE_THRESHOLD,
      duration: latestResult.duration <= MAX_DURATION,
      noInfraFailures: (metrics.failureCategories.infrastructure || 0) === 0
    };

    console.log('\n  🔍 Readiness Checks:');
    console.log(`    ${checks.passRate ? '✅' : '❌'} Pass rate ≥ ${PASS_RATE_THRESHOLD}% (${latestResult.passRate.toFixed(2)}%)`);
    console.log(`    ${checks.coverage ? '✅' : '❌'} Coverage ≥ ${COVERAGE_THRESHOLD}% (${metrics.coverage.toFixed(2)}%)`);
    console.log(`    ${checks.duration ? '✅' : '✅'} Duration < 5 min (${(latestResult.duration / 1000).toFixed(2)}s)`);
    console.log(`    ${checks.noInfraFailures ? '✅' : '⚠️ '} No critical infrastructure failures`);

    const allChecksPassed = Object.values(checks).every(c => c);
    const recommendation = allChecksPassed ? 'GO' : 'NO-GO';
    const safetyNetScore = this.calculateSafetyScore(latestResult.passRate, metrics.coverage);

    const validation: RegressionValidation = {
      timestamp: Date.now(),
      agent: 'regression-validator',
      passRate: latestResult.passRate,
      coverage: metrics.coverage,
      stability: metrics.trend,
      recommendation,
      regressionSuiteReady: allChecksPassed,
      safetyNetScore,
      metrics
    };

    // Store validation
    await this.memoryStore.store('aqe/regression/validation', validation, {
      partition: 'coordination',
      ttl: 86400
    });

    await this.memoryStore.store('aqe/regression/final-validation', validation, {
      partition: 'coordination',
      ttl: 86400
    });

    await this.memoryStore.storePerformanceMetric({
      metric: 'regression_pass_rate',
      value: latestResult.passRate,
      unit: 'percentage',
      timestamp: Date.now()
    });

    await this.memoryStore.storePerformanceMetric({
      metric: 'safety_net_score',
      value: safetyNetScore,
      unit: 'score',
      timestamp: Date.now()
    });

    console.log(`\n  🎯 Safety Net Score: ${safetyNetScore.toFixed(2)}/100`);
    console.log(`  🚦 Recommendation: ${recommendation}`);

    return validation;
  }

  /**
   * Calculate safety net score (0-100)
   */
  private calculateSafetyScore(passRate: number, coverage: number): number {
    // Weighted score: 60% pass rate, 40% coverage
    const passRateScore = (passRate / 100) * 60;
    const coverageScore = (coverage / 100) * 40;
    return Math.min(passRateScore + coverageScore, 100);
  }

  /**
   * Generate stability report
   */
  async generateStabilityReport(): Promise<string> {
    console.log('\n📝 Generating stability report...');

    const latestResult = this.results[this.results.length - 1];
    const metrics = await this.trackStabilityMetrics();
    const validation = await this.validateRegressionSuite();

    const report = `# Regression Validation Report

**Generated:** ${new Date().toISOString()}
**Validator:** Regression Validation Specialist
**Sprint:** Test Stabilization

---

## Executive Summary

- **Recommendation:** \`${validation.recommendation}\`
- **Safety Net Score:** ${validation.safetyNetScore.toFixed(2)}/100
- **Regression Suite Ready:** ${validation.regressionSuiteReady ? '✅ YES' : '❌ NO'}

---

## Test Suite Metrics

### Latest Run (Run #${this.runCount})

| Metric | Value |
|--------|-------|
| **Pass Rate** | ${latestResult.passRate.toFixed(2)}% |
| **Tests Passed** | ${latestResult.passed} |
| **Tests Failed** | ${latestResult.failed} |
| **Tests Skipped** | ${latestResult.skipped} |
| **Total Tests** | ${latestResult.total} |
| **Duration** | ${(latestResult.duration / 1000).toFixed(2)}s |

### Stability Trend

- **Trend:** ${metrics.trend}
- **Consecutive Runs:** ${metrics.consecutiveRuns}
- **Average Pass Rate (Last 5):** ${metrics.passRate.toFixed(2)}%
- **Coverage Estimate:** ${metrics.coverage.toFixed(2)}%

---

## Pass Rate Over Time

\`\`\`
${this.generatePassRateChart()}
\`\`\`

---

## Failure Analysis

### Failure Categories

| Category | Count |
|----------|-------|
${Object.entries(metrics.failureCategories)
  .map(([cat, count]) => `| **${cat}** | ${count} |`)
  .join('\n')}

### Failure Reduction Metrics

${this.generateFailureReduction()}

---

## Readiness Validation

### Criteria

| Criteria | Threshold | Actual | Status |
|----------|-----------|--------|--------|
| **Pass Rate** | ≥ 70% | ${latestResult.passRate.toFixed(2)}% | ${latestResult.passRate >= 70 ? '✅' : '❌'} |
| **Coverage** | ≥ 15% | ${metrics.coverage.toFixed(2)}% | ${metrics.coverage >= 15 ? '✅' : '❌'} |
| **Duration** | < 5 min | ${(latestResult.duration / 1000).toFixed(2)}s | ${latestResult.duration <= 300000 ? '✅' : '❌'} |
| **Infrastructure** | No Critical | ${metrics.failureCategories.infrastructure || 0} issues | ${(metrics.failureCategories.infrastructure || 0) === 0 ? '✅' : '⚠️'} |

---

## Safety Net Assessment

### Score Breakdown

- **Pass Rate Component (60%):** ${((latestResult.passRate / 100) * 60).toFixed(2)} points
- **Coverage Component (40%):** ${((metrics.coverage / 100) * 40).toFixed(2)} points
- **Total Score:** ${validation.safetyNetScore.toFixed(2)}/100

### Interpretation

${this.interpretSafetyScore(validation.safetyNetScore)}

---

## Recommendations

${validation.recommendation === 'GO' ? `
✅ **GO for Regression Suite Deployment**

The regression suite has met all readiness criteria:
- Pass rate is above threshold (${latestResult.passRate.toFixed(2)}% ≥ 70%)
- Coverage is adequate (${metrics.coverage.toFixed(2)}% ≥ 15%)
- Test suite runs efficiently (${(latestResult.duration / 1000).toFixed(2)}s < 5 min)
- Trend is ${metrics.trend.toLowerCase()}

**Next Steps:**
1. Deploy regression suite to CI/CD pipeline
2. Enable automated regression checks on PRs
3. Set up failure alerting
4. Continue monitoring stability metrics
` : `
❌ **NO-GO for Regression Suite Deployment**

The regression suite does not meet readiness criteria:
${latestResult.passRate < 70 ? `- ⚠️  Pass rate is below threshold (${latestResult.passRate.toFixed(2)}% < 70%)` : ''}
${metrics.coverage < 15 ? `- ⚠️  Coverage is below threshold (${metrics.coverage.toFixed(2)}% < 15%)` : ''}

**Required Actions:**
1. Address failing tests to improve pass rate
2. Increase test coverage to meet minimum threshold
3. Fix infrastructure issues if any
4. Re-run validation after fixes
`}

---

## Agent Coordination

### Monitored Tasks

${await this.getAgentCoordinationStatus()}

---

**Report Generated by Regression Validation Specialist**
**Timestamp:** ${Date.now()}
`;

    return report;
  }

  /**
   * Generate pass rate chart (text-based)
   */
  private generatePassRateChart(): string {
    if (this.results.length === 0) return 'No data available';

    const lines: string[] = [];
    lines.push('Run # | Pass Rate | Chart');
    lines.push('------|-----------|------');

    this.results.forEach((result, idx) => {
      const barLength = Math.round((result.passRate / 100) * 40);
      const bar = '█'.repeat(barLength) + '░'.repeat(40 - barLength);
      lines.push(`${(idx + 1).toString().padStart(5)} | ${result.passRate.toFixed(2).padStart(6)}% | ${bar}`);
    });

    return lines.join('\n');
  }

  /**
   * Generate failure reduction metrics
   */
  private generateFailureReduction(): string {
    if (this.results.length < 2) return 'Not enough data for comparison';

    const firstResult = this.results[0];
    const latestResult = this.results[this.results.length - 1];

    const failureReduction = firstResult.failed - latestResult.failed;
    const percentReduction = firstResult.failed > 0
      ? ((failureReduction / firstResult.failed) * 100).toFixed(2)
      : '0.00';

    return `
- **Initial Failures:** ${firstResult.failed}
- **Current Failures:** ${latestResult.failed}
- **Reduction:** ${failureReduction} tests (${percentReduction}%)
- **Improvement:** ${latestResult.passRate > firstResult.passRate ? '✅ Improving' : '⚠️ Needs attention'}
`;
  }

  /**
   * Interpret safety score
   */
  private interpretSafetyScore(score: number): string {
    if (score >= 90) return '🟢 **EXCELLENT** - Regression suite is highly reliable and ready for production use.';
    if (score >= 70) return '🟡 **GOOD** - Regression suite is ready with acceptable reliability.';
    if (score >= 50) return '🟠 **FAIR** - Regression suite needs improvement before deployment.';
    return '🔴 **POOR** - Regression suite requires significant work before deployment.';
  }

  /**
   * Get agent coordination status
   */
  private async getAgentCoordinationStatus(): Promise<string> {
    try {
      const monitoring = await this.memoryStore.retrieve('aqe/regression/agent-monitoring', {
        partition: 'coordination'
      });

      if (!monitoring) return '⏳ Waiting for other agents to start...';

      return `
- **Infrastructure Fixes:** ${monitoring.infraFixes ? '✅ Active' : '⏳ Pending'}
- **Test Fixes:** ${monitoring.testFixes ? '✅ Active' : '⏳ Pending'}
- **Coverage Tracking:** ${monitoring.coverageTracked ? '✅ Active' : '⏳ Pending'}
`;
    } catch {
      return '⏳ No coordination data available yet';
    }
  }

  /**
   * Main validation loop
   */
  async run(iterations: number = 3, intervalMinutes: number = 2): Promise<void> {
    console.log('🚀 Starting Regression Validation Specialist');
    console.log(`📊 Will run ${iterations} test cycles with ${intervalMinutes} min intervals\n`);

    await this.initialize();

    for (let i = 0; i < iterations; i++) {
      console.log(`\n${'='.repeat(80)}`);
      console.log(`Iteration ${i + 1}/${iterations}`);
      console.log('='.repeat(80));

      // Monitor other agents
      await this.monitorAgentProgress();

      // Run regression tests
      await this.runRegressionTests();

      // Track metrics
      await this.trackStabilityMetrics();

      // Wait before next iteration (except on last iteration)
      if (i < iterations - 1) {
        console.log(`\n⏳ Waiting ${intervalMinutes} minutes before next run...`);
        await new Promise(resolve => setTimeout(resolve, intervalMinutes * 60 * 1000));
      }
    }

    // Final validation
    console.log(`\n${'='.repeat(80)}`);
    console.log('FINAL VALIDATION');
    console.log('='.repeat(80));

    const validation = await this.validateRegressionSuite();

    // Generate report
    const report = await this.generateStabilityReport();
    const reportPath = path.join(process.cwd(), 'docs/reports/REGRESSION-VALIDATION.md');
    fs.writeFileSync(reportPath, report);

    console.log(`\n✅ Validation complete!`);
    console.log(`📄 Report saved to: ${reportPath}`);
    console.log(`\n🚦 FINAL RECOMMENDATION: ${validation.recommendation}`);
    console.log(`🎯 Safety Net Score: ${validation.safetyNetScore.toFixed(2)}/100`);
  }
}

// Run if executed directly
if (require.main === module) {
  const validator = new RegressionValidator();
  const iterations = parseInt(process.argv[2] || '3', 10);
  const interval = parseInt(process.argv[3] || '2', 10);

  validator.run(iterations, interval).catch(error => {
    console.error('❌ Validation failed:', error);
    process.exit(1);
  });
}

export { RegressionValidator };