proffesor-for-testing__agentic-qe/tests/performance/cli-performance.bench.ts

/**
 * CLI Performance Benchmarks
 * ADR-041: V3 QE CLI Enhancement Performance Targets
 *
 * Performance targets from ADR-041:
 * | Metric                | Current | Target   |
 * |-----------------------|---------|----------|
 * | Startup time          | ~1.2s   | <400ms   |
 * | Completion response   | ~300ms  | <50ms    |
 * | Progress update rate  | 1/s     | 10/s     |
 * | Wizard navigation     | ~200ms  | <50ms/step |
 *
 * Run with: npm run test:perf
 */

import { describe, bench, afterAll } from 'vitest';
import { spawn, type ChildProcess } from 'child_process';
import { join } from 'path';

// Import modules at top level to avoid beforeAll timing issues
import {
  generateBashCompletion,
  generateZshCompletion,
  generateFishCompletion,
  generatePowerShellCompletion,
  generateCompletion,
  detectShell,
} from '../../src/cli/completions/index.js';

import { EtaEstimator } from '../../src/cli/utils/progress.js';

import {
  TestGenerationWizard,
} from '../../src/cli/wizards/test-wizard.js';

import {
  CoverageAnalysisWizard,
  getSensitivityConfig,
} from '../../src/cli/wizards/coverage-wizard.js';

// ============================================================================
// Performance Targets (ADR-041)
// ============================================================================

const PERFORMANCE_TARGETS = {
  /** CLI startup time target in milliseconds */
  startupTime: 400,
  /** Completion generation time target in milliseconds */
  completionResponse: 50,
  /** Progress update interval target in milliseconds (10/s = 100ms) */
  progressUpdateInterval: 100,
  /** Wizard step navigation time target in milliseconds */
  wizardNavigation: 50,
} as const;

// ============================================================================
// Helper: Measure CLI Startup Time via Spawn
// ============================================================================

interface StartupMeasurement {
  durationMs: number;
  exitCode: number | null;
  stdout: string;
  stderr: string;
}

/**
 * Measure actual CLI startup time by spawning a new process
 * This measures the real-world startup experience
 */
async function measureCLIStartup(args: string[] = ['--help']): Promise<StartupMeasurement> {
  const cliPath = join(__dirname, '../../src/cli/index.ts');
  const startTime = performance.now();

  return new Promise((resolve) => {
    let stdout = '';
    let stderr = '';

    const proc: ChildProcess = spawn('npx', ['tsx', cliPath, ...args], {
      cwd: join(__dirname, '../..'),
      stdio: ['pipe', 'pipe', 'pipe'],
      env: { ...process.env, NODE_ENV: 'test' },
    });

    proc.stdout?.on('data', (data) => {
      stdout += data.toString();
    });

    proc.stderr?.on('data', (data) => {
      stderr += data.toString();
    });

    proc.on('close', (exitCode) => {
      const endTime = performance.now();
      resolve({
        durationMs: endTime - startTime,
        exitCode,
        stdout,
        stderr,
      });
    });

    proc.on('error', () => {
      const endTime = performance.now();
      resolve({
        durationMs: endTime - startTime,
        exitCode: -1,
        stdout,
        stderr,
      });
    });

    // Timeout after 5 seconds
    setTimeout(() => {
      proc.kill('SIGTERM');
    }, 5000);
  });
}

// ============================================================================
// Helper: Measure Import Time (Module Loading)
// ============================================================================

async function measureModuleImport(modulePath: string): Promise<number> {
  const startTime = performance.now();
  await import(modulePath);
  return performance.now() - startTime;
}

// ============================================================================
// Benchmark: CLI Startup Time (ADR-041 Target: <400ms)
// ============================================================================

describe('CLI Startup Time (ADR-041 Target: <400ms)', () => {
  bench('CLI startup via spawn with --help', async () => {
    await measureCLIStartup(['--help']);
  }, {
    iterations: 3,
    warmupIterations: 1,
  });

  bench('CLI module import time', async () => {
    // Force fresh import by cache busting
    const modulePath = `../../src/cli/index.ts?t=${Date.now()}`;
    try {
      await import(modulePath);
    } catch {
      // Expected - file doesn't exist with cache buster, but measures import system overhead
    }
  }, {
    iterations: 5,
  });
});

// ============================================================================
// Benchmark: Completion Generation (ADR-041 Target: <50ms)
// ============================================================================

describe('Completion Generation (ADR-041 Target: <50ms)', () => {
  bench('generateBashCompletion()', () => {
    const script = generateBashCompletion();
    if (!script || script.length < 100) {
      throw new Error('Invalid bash completion script');
    }
  }, {
    iterations: 100,
    warmupIterations: 10,
  });

  bench('generateZshCompletion()', () => {
    const script = generateZshCompletion();
    if (!script || script.length < 100) {
      throw new Error('Invalid zsh completion script');
    }
  }, {
    iterations: 100,
    warmupIterations: 10,
  });

  bench('generateFishCompletion()', () => {
    const script = generateFishCompletion();
    if (!script || script.length < 100) {
      throw new Error('Invalid fish completion script');
    }
  }, {
    iterations: 100,
    warmupIterations: 10,
  });

  bench('generatePowerShellCompletion()', () => {
    const script = generatePowerShellCompletion();
    if (!script || script.length < 100) {
      throw new Error('Invalid PowerShell completion script');
    }
  }, {
    iterations: 100,
    warmupIterations: 10,
  });

  bench('generateCompletion() - all shells', () => {
    const shells = ['bash', 'zsh', 'fish', 'powershell'] as const;
    for (const shell of shells) {
      generateCompletion(shell);
    }
  }, {
    iterations: 50,
    warmupIterations: 5,
  });

  bench('detectShell()', () => {
    detectShell();
  }, {
    iterations: 1000,
    warmupIterations: 100,
  });
});

// ============================================================================
// Benchmark: Progress Bar Updates (ADR-041 Target: 10 updates/second)
// ============================================================================

describe('Progress Bar Updates (ADR-041 Target: 10/s = 100ms/update)', () => {
  bench('EtaEstimator.addSample() + estimate()', () => {
    const estimator = new EtaEstimator(10);
    // Simulate progress updates
    for (let i = 0; i <= 100; i += 10) {
      estimator.addSample(i);
      estimator.estimate(i);
    }
  }, {
    iterations: 100,
    warmupIterations: 10,
  });

  bench('FleetProgressManager state update cycle', () => {
    // Measure the internal state update logic without actual console output
    const agentState = {
      id: 'agent-1',
      name: 'test-agent',
      status: 'running' as const,
      progress: 50,
      eta: 1000,
    };

    // Simulate the update calculations that would happen in updateAgent
    const newProgress = Math.min(100, Math.max(0, agentState.progress + 10));
    const statusIcon = agentState.status === 'completed' ? '\u2713' :
                       agentState.status === 'failed' ? '\u2717' :
                       agentState.status === 'running' ? '\u25B6' : '\u25CB';

    // Format ETA
    const formatEta = (ms: number): string => {
      if (ms <= 0) return '';
      if (ms < 1000) return `${ms}ms`;
      if (ms < 60000) return `${Math.round(ms / 1000)}s`;
      return `${Math.round(ms / 60000)}m`;
    };

    formatEta(agentState.eta);
  }, {
    iterations: 10000,
    warmupIterations: 1000,
  });

  bench('Progress update simulation (10 updates)', () => {
    // Simulate what happens during 10 progress updates
    const updates: Array<{ progress: number; eta: number }> = [];

    for (let i = 0; i < 10; i++) {
      updates.push({
        progress: i * 10,
        eta: (10 - i) * 100,
      });
    }

    // Process updates
    for (const update of updates) {
      const _formatted = update.eta < 1000 ? `${update.eta}ms` : `${Math.round(update.eta / 1000)}s`;
      const _percentage = update.progress.toFixed(1);
    }
  }, {
    iterations: 1000,
    warmupIterations: 100,
  });
});

// ============================================================================
// Benchmark: Wizard Step Navigation (ADR-041 Target: <50ms/step)
// ============================================================================

describe('Wizard Step Navigation (ADR-041 Target: <50ms/step)', () => {
  bench('TestGenerationWizard instantiation', () => {
    new TestGenerationWizard({ nonInteractive: true });
  }, {
    iterations: 1000,
    warmupIterations: 100,
  });

  bench('TestGenerationWizard.run() - non-interactive mode', async () => {
    const wizard = new TestGenerationWizard({ nonInteractive: true });
    await wizard.run();
  }, {
    iterations: 100,
    warmupIterations: 10,
  });

  bench('CoverageAnalysisWizard instantiation', () => {
    new CoverageAnalysisWizard({ nonInteractive: true });
  }, {
    iterations: 1000,
    warmupIterations: 100,
  });

  bench('CoverageAnalysisWizard.run() - non-interactive mode', async () => {
    const wizard = new CoverageAnalysisWizard({ nonInteractive: true });
    await wizard.run();
  }, {
    iterations: 100,
    warmupIterations: 10,
  });

  bench('getSensitivityConfig() lookup', () => {
    const sensitivities = ['low', 'medium', 'high'] as const;
    for (const s of sensitivities) {
      getSensitivityConfig(s);
    }
  }, {
    iterations: 10000,
    warmupIterations: 1000,
  });

  bench('Wizard state transition simulation', () => {
    // Simulate the state transitions that happen during wizard navigation
    const states = ['sourceFiles', 'testType', 'coverage', 'framework', 'aiLevel', 'antiPatterns'];
    let currentStep = 0;

    // Forward navigation
    for (let i = 0; i < states.length; i++) {
      currentStep = i;
      const _stepData = {
        step: currentStep + 1,
        total: states.length,
        name: states[currentStep],
        isFirst: currentStep === 0,
        isLast: currentStep === states.length - 1,
      };
    }

    // Backward navigation
    for (let i = states.length - 1; i >= 0; i--) {
      currentStep = i;
    }
  }, {
    iterations: 10000,
    warmupIterations: 1000,
  });
});

// ============================================================================
// Performance Validation Tests (Fail if Targets Not Met)
// ============================================================================

describe('ADR-041 Performance Target Validation', () => {
  bench('Completion generation must be <50ms', () => {
    const start = performance.now();
    generateBashCompletion();
    const elapsed = performance.now() - start;

    if (elapsed > PERFORMANCE_TARGETS.completionResponse) {
      throw new Error(
        `PERFORMANCE FAILURE: Completion generation took ${elapsed.toFixed(2)}ms, ` +
        `target is <${PERFORMANCE_TARGETS.completionResponse}ms`
      );
    }
  }, {
    iterations: 20,
    warmupIterations: 5,
  });

  bench('Wizard instantiation must be <50ms', () => {
    const start = performance.now();
    new TestGenerationWizard({ nonInteractive: true });
    const elapsed = performance.now() - start;

    if (elapsed > PERFORMANCE_TARGETS.wizardNavigation) {
      throw new Error(
        `PERFORMANCE FAILURE: Wizard instantiation took ${elapsed.toFixed(2)}ms, ` +
        `target is <${PERFORMANCE_TARGETS.wizardNavigation}ms`
      );
    }
  }, {
    iterations: 20,
    warmupIterations: 5,
  });

  bench('Progress update cycle must support 10/s rate', () => {
    const estimator = new EtaEstimator(10);

    const start = performance.now();
    // Perform 10 updates (1 second worth at target rate)
    for (let i = 0; i <= 100; i += 10) {
      estimator.addSample(i);
      estimator.estimate(i);
    }
    const elapsed = performance.now() - start;
    const perUpdateMs = elapsed / 10;

    if (perUpdateMs > PERFORMANCE_TARGETS.progressUpdateInterval) {
      throw new Error(
        `PERFORMANCE FAILURE: Progress update took ${perUpdateMs.toFixed(2)}ms/update, ` +
        `target is <${PERFORMANCE_TARGETS.progressUpdateInterval}ms/update for 10/s rate`
      );
    }
  }, {
    iterations: 20,
    warmupIterations: 5,
  });
});

// ============================================================================
// Performance Summary Report
// ============================================================================

describe('Performance Summary', () => {
  afterAll(() => {
    console.log('\n');
    console.log('='.repeat(70));
    console.log('  ADR-041 CLI Performance Benchmark Results');
    console.log('='.repeat(70));
    console.log('');
    console.log('  Performance Targets:');
    console.log(`    - Startup time:        <${PERFORMANCE_TARGETS.startupTime}ms`);
    console.log(`    - Completion response: <${PERFORMANCE_TARGETS.completionResponse}ms`);
    console.log(`    - Progress updates:    ${1000 / PERFORMANCE_TARGETS.progressUpdateInterval}/s (${PERFORMANCE_TARGETS.progressUpdateInterval}ms/update)`);
    console.log(`    - Wizard navigation:   <${PERFORMANCE_TARGETS.wizardNavigation}ms/step`);
    console.log('');
    console.log('  See individual benchmark results above for actual measurements.');
    console.log('  Benchmarks that fail targets will throw errors.');
    console.log('='.repeat(70));
    console.log('');
  });

  bench('placeholder for summary output', () => {
    // This bench exists just to trigger afterAll
  }, { iterations: 1 });
});

// ============================================================================
// Exports for programmatic access
// ============================================================================

export {
  PERFORMANCE_TARGETS,
  measureCLIStartup,
  measureModuleImport,
};