proffesor-for-testing__agentic-qe/tests/unit/governance/ab-benchmarking.test.ts

/**
 * Unit tests for governance/ab-benchmarking.ts
 *
 * Tests: benchmark creation, outcome recording, statistical significance,
 * winner determination, and validation.
 */
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';

// Mock dependencies
vi.mock('../../../src/governance/feature-flags.js', () => ({
  governanceFlags: {
    getFlags: vi.fn().mockReturnValue({
      global: { enableAllGates: true, logViolations: false },
      abBenchmarking: {
        enabled: true,
        defaultConfidenceLevel: 0.95,
        defaultMinSampleSize: 100,
        autoApplyWinners: false,
        maxConcurrentBenchmarks: 5,
      },
    }),
  },
}));

vi.mock('../../../src/governance/evolution-pipeline-integration.js', () => ({
  evolutionPipelineIntegration: {
    initialize: vi.fn().mockResolvedValue(undefined),
    registerVariantTest: vi.fn(),
    recordVariantOutcome: vi.fn(),
    completeVariantTest: vi.fn(),
    promoteRule: vi.fn(),
  },
}));

import {
  ABBenchmarkingFramework,
  createBenchmarkConfig,
  type BenchmarkConfig,
  type VariantConfig,
} from '../../../src/governance/ab-benchmarking.js';

describe('ABBenchmarkingFramework', () => {
  let framework: ABBenchmarkingFramework;

  const variants: VariantConfig[] = [
    { id: 'control', name: 'Control', rules: { threshold: 0.5 } },
    { id: 'treatment', name: 'Treatment', rules: { threshold: 0.7 } },
  ];

  const defaultConfig: BenchmarkConfig = {
    testId: 'test-1',
    variants,
    metrics: [
      { name: 'success_rate', type: 'success_rate', weight: 0.5, higherIsBetter: true },
      { name: 'latency', type: 'latency', weight: 0.3, higherIsBetter: false },
      { name: 'quality', type: 'quality_score', weight: 0.2, higherIsBetter: true },
    ],
    minSampleSize: 5,
    confidenceLevel: 0.95,
    maxDurationMs: 60_000,
  };

  beforeEach(async () => {
    framework = new ABBenchmarkingFramework();
    await framework.initialize();
  });

  afterEach(() => {
    vi.restoreAllMocks();
  });

  // ============================================================================
  // Benchmark Creation
  // ============================================================================

  describe('createBenchmark', () => {
    it('should create a benchmark and return its id', () => {
      const id = framework.createBenchmark(defaultConfig);
      expect(id).toBe('test-1');
    });

    it('should reject benchmark with fewer than 2 variants', () => {
      expect(() =>
        framework.createBenchmark({
          ...defaultConfig,
          variants: [variants[0]],
        })
      ).toThrow('at least 2 variants');
    });

    it('should reject benchmark with no metrics', () => {
      expect(() =>
        framework.createBenchmark({
          ...defaultConfig,
          metrics: [],
        })
      ).toThrow('at least 1 metric');
    });

    it('should reject benchmark with metric weights not summing to 1', () => {
      expect(() =>
        framework.createBenchmark({
          ...defaultConfig,
          metrics: [
            { name: 'success_rate', type: 'success_rate', weight: 0.3, higherIsBetter: true },
            { name: 'latency', type: 'latency', weight: 0.3, higherIsBetter: false },
          ],
        })
      ).toThrow('weights must sum to 1.0');
    });

    it('should reject benchmark when max concurrent limit reached', () => {
      for (let i = 0; i < 5; i++) {
        framework.createBenchmark({ ...defaultConfig, testId: `test-${i}` });
      }
      expect(() =>
        framework.createBenchmark({ ...defaultConfig, testId: 'test-excess' })
      ).toThrow('Maximum concurrent benchmarks');
    });
  });

  // ============================================================================
  // Benchmark Lifecycle
  // ============================================================================

  describe('startBenchmark / stopBenchmark', () => {
    it('should start a pending benchmark', () => {
      framework.createBenchmark(defaultConfig);
      framework.startBenchmark('test-1');

      const active = framework.getActiveBenchmarks();
      expect(active.some(b => b.benchmarkId === 'test-1' && b.status === 'running')).toBe(true);
    });

    it('should throw for non-existent benchmark', () => {
      expect(() => framework.startBenchmark('nonexistent')).toThrow('not found');
    });

    it('should throw when starting a completed benchmark', () => {
      framework.createBenchmark(defaultConfig);
      framework.startBenchmark('test-1');
      framework.stopBenchmark('test-1');

      // Stopped benchmarks cannot be restarted as 'completed' either
      // stopBenchmark sets status to 'stopped', which is not 'completed'
      // but we can verify the stop worked
      const history = framework.getBenchmarkHistory();
      expect(history.some(b => b.benchmarkId === 'test-1' && b.status === 'stopped')).toBe(true);
    });

    it('should stop a running benchmark', () => {
      framework.createBenchmark(defaultConfig);
      framework.startBenchmark('test-1');
      framework.stopBenchmark('test-1');

      const active = framework.getActiveBenchmarks();
      expect(active.find(b => b.benchmarkId === 'test-1')).toBeUndefined();
    });
  });

  // ============================================================================
  // Recording Outcomes
  // ============================================================================

  describe('recordOutcome', () => {
    beforeEach(() => {
      framework.createBenchmark(defaultConfig);
      framework.startBenchmark('test-1');
    });

    it('should record success outcomes', () => {
      framework.recordOutcome('test-1', 'control', true, { latency: 100, quality: 0.9 });

      const results = framework.getBenchmarkResults('test-1');
      const control = results.variants.find(v => v.id === 'control');
      expect(control!.sampleSize).toBe(1);
      expect(control!.successRate).toBe(1);
    });

    it('should record failure outcomes', () => {
      framework.recordOutcome('test-1', 'control', false);

      const results = framework.getBenchmarkResults('test-1');
      const control = results.variants.find(v => v.id === 'control');
      expect(control!.sampleSize).toBe(1);
      expect(control!.successRate).toBe(0);
    });

    it('should accumulate multiple outcomes', () => {
      for (let i = 0; i < 5; i++) {
        framework.recordOutcome('test-1', 'control', true, { latency: 100 + i * 10, quality: 0.8 });
        framework.recordOutcome('test-1', 'treatment', i < 4, { latency: 80 + i * 10, quality: 0.85 });
      }

      const results = framework.getBenchmarkResults('test-1');
      expect(results.variants.find(v => v.id === 'control')!.sampleSize).toBe(5);
      expect(results.variants.find(v => v.id === 'treatment')!.sampleSize).toBe(5);
    });
  });

  // ============================================================================
  // recordMetric
  // ============================================================================

  describe('recordMetric', () => {
    beforeEach(() => {
      framework.createBenchmark(defaultConfig);
      framework.startBenchmark('test-1');
    });

    it('should record individual metric values', () => {
      framework.recordMetric('test-1', 'control', 'latency', 150);
      framework.recordMetric('test-1', 'control', 'latency', 200);

      const results = framework.getBenchmarkResults('test-1');
      const control = results.variants.find(v => v.id === 'control');
      expect(control!.metrics['latency'].mean).toBe(175);
    });
  });

  // ============================================================================
  // Statistical Analysis
  // ============================================================================

  describe('calculateStatisticalSignificance', () => {
    it('should calculate significance with enough samples', () => {
      framework.createBenchmark({ ...defaultConfig, minSampleSize: 3 });
      framework.startBenchmark('test-1');

      // Record enough data with clear difference
      for (let i = 0; i < 10; i++) {
        framework.recordOutcome('test-1', 'control', i < 5, { latency: 200, quality: 0.7 });
        framework.recordOutcome('test-1', 'treatment', i < 9, { latency: 100, quality: 0.9 });
      }

      const sig = framework.calculateStatisticalSignificance('test-1');
      expect(sig.benchmarkId).toBe('test-1');
      expect(sig.confidenceLevel).toBe(0.95);
      expect(sig.bonferroniAlpha).toBeGreaterThan(0);
      expect(sig.chiSquareTest).not.toBeNull();
      expect(sig.powerAnalysis.targetPower).toBe(0.8);
    });
  });

  // ============================================================================
  // Winner Determination
  // ============================================================================

  describe('getWinner', () => {
    it('should return null when not enough samples', () => {
      framework.createBenchmark(defaultConfig);
      framework.startBenchmark('test-1');
      framework.recordOutcome('test-1', 'control', true);

      const winner = framework.getWinner('test-1');
      expect(winner).toBeNull();
    });

    it('should determine winner when enough samples exist', () => {
      framework.createBenchmark({ ...defaultConfig, minSampleSize: 3 });
      framework.startBenchmark('test-1');

      // Treatment is clearly better
      for (let i = 0; i < 5; i++) {
        framework.recordOutcome('test-1', 'control', i < 2, { latency: 300, quality: 0.5 });
        framework.recordOutcome('test-1', 'treatment', true, { latency: 100, quality: 0.95 });
      }

      const winner = framework.getWinner('test-1');
      expect(winner).not.toBeNull();
      expect(winner!.winnerId).toBe('treatment');
      expect(winner!.recommendation).toBeDefined();
    });
  });

  // ============================================================================
  // Variant Comparison
  // ============================================================================

  describe('compareVariants', () => {
    it('should compare two variants with metric breakdowns', () => {
      framework.createBenchmark(defaultConfig);
      framework.startBenchmark('test-1');

      for (let i = 0; i < 5; i++) {
        framework.recordOutcome('test-1', 'control', true, { latency: 200, quality: 0.7 });
        framework.recordOutcome('test-1', 'treatment', true, { latency: 100, quality: 0.9 });
      }

      const comparison = framework.compareVariants('test-1', 'control', 'treatment');
      expect(comparison.benchmarkId).toBe('test-1');
      expect(comparison.metricComparisons.length).toBeGreaterThan(0);
      expect(comparison.combinedScores.variantA).toBeDefined();
      expect(comparison.combinedScores.variantB).toBeDefined();
    });

    it('should throw for non-existent variant', () => {
      framework.createBenchmark(defaultConfig);
      expect(() =>
        framework.compareVariants('test-1', 'control', 'nonexistent')
      ).toThrow('Variant not found');
    });
  });

  // ============================================================================
  // Suggestion
  // ============================================================================

  describe('suggestWinner', () => {
    it('should add caveats when insufficient samples', () => {
      framework.createBenchmark(defaultConfig);
      framework.startBenchmark('test-1');

      framework.recordOutcome('test-1', 'control', true);
      framework.recordOutcome('test-1', 'treatment', true);

      const suggestion = framework.suggestWinner('test-1');
      expect(suggestion.caveats.some(c => c.includes('Insufficient samples'))).toBe(true);
      expect(suggestion.readyToApply).toBe(false);
    });
  });

  // ============================================================================
  // Helper: createBenchmarkConfig
  // ============================================================================

  describe('createBenchmarkConfig', () => {
    it('should create config with default metrics when none provided', () => {
      const config = createBenchmarkConfig('test-default', variants);
      expect(config.metrics).toHaveLength(3);
      expect(config.testId).toBe('test-default');
      expect(config.variants).toBe(variants);
    });

    it('should use flag defaults for minSampleSize and confidenceLevel', () => {
      const config = createBenchmarkConfig('test-defaults', variants);
      expect(config.minSampleSize).toBe(100);
      expect(config.confidenceLevel).toBe(0.95);
    });

    it('should allow overriding minSampleSize', () => {
      const config = createBenchmarkConfig('test-override', variants, {
        minSampleSize: 50,
      });
      expect(config.minSampleSize).toBe(50);
    });
  });

  // ============================================================================
  // Reset
  // ============================================================================

  describe('reset', () => {
    it('should clear all benchmarks', () => {
      framework.createBenchmark(defaultConfig);
      framework.reset();
      expect(framework.getBenchmarkHistory()).toHaveLength(0);
    });
  });
});