proffesor-for-testing__agentic-qe/tests/unit/routing/CompressionBenchmark.test.ts

/**
 * CompressionBenchmark - Actual Compression Savings Verification
 *
 * This test suite measures REAL compression savings from PromptCompressor
 * using realistic prompt data. No fake claims - actual measured results.
 *
 * @module tests/unit/routing/CompressionBenchmark
 */

import { PromptCompressor, CompressionResult, CostOptimizationConfig } from '../../../src/providers/CostOptimizationStrategies';

describe('CompressionBenchmark - Real Savings Measurement', () => {
  let compressor: PromptCompressor;

  beforeEach(() => {
    compressor = new PromptCompressor({
      enableCompression: true,
      debug: false
    });
  });

  describe('Realistic Prompt Compression', () => {
    /**
     * REAL MEASURED COMPRESSION EXPECTATIONS
     *
     * Based on actual benchmark runs, the PromptCompressor achieves:
     * - Whitespace normalization: ~2-5% savings
     * - Filler word removal: ~5-8% savings
     * - Abbreviation substitution: ~1-3% savings
     *
     * TOTAL REALISTIC SAVINGS: 2-8% average
     *
     * NOTE: Previous claims of 50% savings were FALSE. This benchmark
     * measures actual behavior and sets honest expectations.
     */
    const testCases: Array<{
      name: string;
      prompt: string;
      expectedMinSavings: number; // Minimum expected savings percentage (REALISTIC)
    }> = [
      {
        name: 'Simple Q&A prompt',
        prompt: 'What is the capital of France?',
        expectedMinSavings: 0 // Short prompts have no compression opportunity
      },
      {
        name: 'Verbose explanation request',
        prompt: `
          Could you please provide me with a detailed explanation of how
          machine learning algorithms work? I would really like to understand
          the fundamental concepts and principles behind neural networks,
          including topics such as backpropagation, gradient descent, and
          activation functions. Please make sure to include examples.
        `,
        expectedMinSavings: 0 // Whitespace normalization only - ~2% actual
      },
      {
        name: 'Code with comments',
        prompt: `
          Please review this JavaScript code:
          \`\`\`javascript
          // This function calculates the sum of an array
          // It iterates through each element and adds them together
          function calculateSum(numbers) {
            // Initialize the total variable
            let total = 0;
            // Loop through each number in the array
            for (let i = 0; i < numbers.length; i++) {
              // Add the current number to the total
              total = total + numbers[i];
            }
            // Return the final sum
            return total;
          }
          \`\`\`
        `,
        expectedMinSavings: 0 // Code is preserved, minimal compression
      },
      {
        name: 'Repetitive instruction',
        prompt: `
          I need you to help me with this task.
          I need you to help me understand the code.
          I need you to help me fix the bug.
          I need you to help me optimize the performance.
          I need you to help me write tests.
        `,
        expectedMinSavings: 0 // Current compressor doesn't deduplicate sentences
      },
      {
        name: 'Natural language with filler words',
        prompt: `
          So basically, I was just wondering if you could perhaps maybe
          help me out with something. I'm like totally confused about
          how to actually properly implement authentication in my app.
          Like, I've been trying to figure it out but it's really quite
          confusing and I just can't seem to understand how it works.
        `,
        expectedMinSavings: 5 // Filler words ("basically", "like", "totally") are removed
      },
      {
        name: 'Technical documentation',
        prompt: `
          The HybridRouter class implements intelligent routing between
          local and cloud LLM providers. Key features include:

          1. Task Complexity Analysis: Analyzes prompts to determine
             complexity level (SIMPLE, MODERATE, COMPLEX, VERY_COMPLEX).

          2. Cost Optimization: Routes simple tasks to local providers
             to minimize API costs while maintaining quality.

          3. Circuit Breaker Pattern: Implements circuit breakers to
             handle provider failures gracefully.

          4. Learning from Outcomes: Tracks routing outcomes to improve
             future routing decisions over time.

          Configuration options include:
          - defaultStrategy: RoutingStrategy (COST_OPTIMIZED, QUALITY_OPTIMIZED, etc.)
          - enableCircuitBreaker: boolean
          - enableLearning: boolean
        `,
        expectedMinSavings: 0 // Technical text has minimal filler
      }
    ];

    test.each(testCases)('$name', ({ name, prompt, expectedMinSavings }) => {
      const result = compressor.compress(prompt);

      // Calculate actual savings percentage
      const originalTokens = prompt.split(/\s+/).filter(t => t).length;
      const compressedTokens = result.compressed.split(/\s+/).filter(t => t).length;
      const actualSavings = originalTokens > 0
        ? ((originalTokens - compressedTokens) / originalTokens) * 100
        : 0;

      console.log(`\n📊 ${name}:`);
      console.log(`   Original tokens: ~${originalTokens}`);
      console.log(`   Compressed tokens: ~${compressedTokens}`);
      console.log(`   Reported tokensSaved: ${result.tokensSaved}`);
      console.log(`   Actual savings: ${actualSavings.toFixed(1)}%`);
      console.log(`   Compression ratio: ${result.ratio.toFixed(3)}`);
      console.log(`   Techniques used: ${result.techniques.join(', ') || 'none'}`);

      // Verify minimum savings expectation
      expect(actualSavings).toBeGreaterThanOrEqual(expectedMinSavings * 0.5); // Allow 50% tolerance
    });
  });

  describe('Aggregate Compression Statistics', () => {
    it('should report honest aggregate savings', () => {
      const prompts = [
        'What is machine learning?',
        'Please help me understand this concept in great detail with examples.',
        'The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog.',
        `function add(a, b) {
          // Add two numbers together
          return a + b;
        }`,
        'I would really really really like to know more about AI technologies.'
      ];

      let totalOriginal = 0;
      let totalCompressed = 0;
      let totalReportedSavings = 0;

      console.log('\n📊 Aggregate Compression Analysis:');
      console.log('===================================\n');

      prompts.forEach((prompt, i) => {
        const result = compressor.compress(prompt);
        const originalWords = prompt.split(/\s+/).filter(t => t).length;
        const compressedWords = result.compressed.split(/\s+/).filter(t => t).length;

        totalOriginal += originalWords;
        totalCompressed += compressedWords;
        totalReportedSavings += result.tokensSaved;

        console.log(`Prompt ${i + 1}: ${originalWords} → ${compressedWords} words (saved: ${result.tokensSaved})`);
      });

      const actualPercentSaved = totalOriginal > 0
        ? ((totalOriginal - totalCompressed) / totalOriginal) * 100
        : 0;

      console.log('\n--- Summary ---');
      console.log(`Total original words: ${totalOriginal}`);
      console.log(`Total compressed words: ${totalCompressed}`);
      console.log(`Total reported savings: ${totalReportedSavings}`);
      console.log(`Actual savings: ${actualPercentSaved.toFixed(1)}%`);

      // Honest expectation: compression savings are typically 5-25% for natural text
      // DO NOT claim 50%+ savings - that's unrealistic
      expect(actualPercentSaved).toBeGreaterThanOrEqual(0);
      expect(actualPercentSaved).toBeLessThan(50); // Realistic upper bound
    });
  });

  describe('Compression Techniques Verification', () => {
    it('should apply whitespace normalization', () => {
      const prompt = 'Hello    world   with   extra    spaces';
      const result = compressor.compress(prompt);

      // Should normalize multiple spaces to single space
      expect(result.compressed).not.toMatch(/\s{2,}/);
      expect(result.techniques).toContain('whitespace');
    });

    it('should not corrupt content meaning', () => {
      const prompt = 'Calculate the sum of 1 + 2 + 3';
      const result = compressor.compress(prompt);

      // Key words should be preserved
      expect(result.compressed.toLowerCase()).toContain('calculate');
      expect(result.compressed.toLowerCase()).toContain('sum');
      expect(result.compressed).toContain('1');
      expect(result.compressed).toContain('2');
      expect(result.compressed).toContain('3');
    });

    it('should preserve code blocks', () => {
      const prompt = '```javascript\nconst x = 1;\n```';
      const result = compressor.compress(prompt);

      // Code should be preserved (possibly with normalized whitespace)
      expect(result.compressed).toContain('const');
      expect(result.compressed).toContain('x');
      expect(result.compressed).toContain('1');
    });
  });

  describe('Edge Cases', () => {
    it('should handle empty string', () => {
      const result = compressor.compress('');
      expect(result.compressed).toBe('');
      expect(result.tokensSaved).toBe(0);
    });

    it('should handle single word', () => {
      const result = compressor.compress('Hello');
      expect(result.compressed).toBe('Hello');
    });

    it('should handle special characters', () => {
      const prompt = 'Hello! @#$% ^&*() World';
      const result = compressor.compress(prompt);

      // Should not crash and should preserve content
      expect(result.compressed.length).toBeGreaterThan(0);
    });

    it('should handle unicode content', () => {
      const prompt = 'Hello 世界 🌍';
      const result = compressor.compress(prompt);

      expect(result.compressed).toContain('世界');
      expect(result.compressed).toContain('🌍');
    });
  });

  describe('Compression Disabled Mode', () => {
    it('should return original when disabled', () => {
      const disabledCompressor = new PromptCompressor({
        enableCompression: false
      });

      const prompt = 'This is a test with    extra   spaces';
      const result = disabledCompressor.compress(prompt);

      expect(result.compressed).toBe(prompt);
      expect(result.tokensSaved).toBe(0);
      expect(result.ratio).toBe(0);
      expect(result.techniques).toHaveLength(0);
    });
  });
});