proffesor-for-testing__agentic-qe/tests/agentdb/performance-benchmark.test.ts

/**
 * AgentDB Performance Benchmarks
 *
 * Comprehensive benchmarks to verify performance claims:
 * - 150x faster vector search vs baseline
 * - QUIC sync <1ms latency
 * - Batch operations <2ms for 100 patterns
 *
 * @version 1.0.0
 * @author Agentic QE Team
 */

import { describe, it, expect, beforeAll, afterAll } from '@jest/globals';
import { EnhancedAgentDBService } from '@core/memory/EnhancedAgentDBService';
import { QEPattern } from '@core/memory/AgentDBService';
import * as fs from 'fs';
import * as path from 'path';
import { createSeededRandom } from '../../src/utils/SeededRandom';

describe('AgentDB Performance Benchmarks', () => {
  const testDbPath = path.join(__dirname, '../fixtures/bench-agentdb.db');
  let agentDB: EnhancedAgentDBService;

  beforeAll(async () => {
    // Clean up existing database
    if (fs.existsSync(testDbPath)) {
      fs.unlinkSync(testDbPath);
    }

    // Initialize AgentDB with HNSW for maximum performance
    agentDB = new EnhancedAgentDBService({
      dbPath: testDbPath,
      embeddingDim: 384,
      enableHNSW: true,
      enableCache: true,
      enableQuantization: true,
      quantizationBits: 8,
      enableQuic: true,
      enableLearning: false // Disable for pure performance testing
    });

    await agentDB.initialize();
  });

  afterAll(async () => {
    if (agentDB) {
      await agentDB.close();
    }

    if (fs.existsSync(testDbPath)) {
      fs.unlinkSync(testDbPath);
    }
  });

  describe('Vector Search Performance', () => {
    it('should achieve 150x speedup vs linear search (10K vectors)', async () => {
      console.log('\n=== Vector Search Performance Test (10K vectors) ===\n');

      const vectorCount = 10000;
      const patterns: QEPattern[] = [];
      const embeddings: number[][] = [];

      // Generate test data
      console.log(`Generating ${vectorCount} test vectors...`);
      for (let i = 0; i < vectorCount; i++) {
        patterns.push({
          id: `pattern-${i}`,
          type: 'test-gen',
          domain: 'test-planning',
          data: { index: i },
          confidence: 0.8,
          usageCount: 0,
          successCount: 0,
          createdAt: Date.now(),
          lastUsed: Date.now()
        });
        embeddings.push(generateRandomEmbedding(384));
      }

      // Batch insert
      console.log('Inserting vectors into AgentDB...');
      const insertStart = Date.now();
      await agentDB.storeBatch(patterns, embeddings);
      const insertDuration = Date.now() - insertStart;
      console.log(`Inserted ${vectorCount} vectors in ${insertDuration}ms (${(vectorCount / insertDuration * 1000).toFixed(0)} vectors/sec)\n`);

      // Benchmark HNSW search
      const queryEmbedding = generateRandomEmbedding(384);
      const k = 10;

      console.log(`Running HNSW search (k=${k})...`);
      const hnswStart = Date.now();
      const hnswResults = await agentDB.searchSimilar(queryEmbedding, { k });
      const hnswDuration = Date.now() - hnswStart;

      expect(hnswResults.length).toBe(k);
      console.log(`HNSW search: ${hnswDuration}ms\n`);

      // Benchmark linear search
      console.log('Running linear search for comparison...');
      const linearStart = Date.now();
      const linearResults = await linearVectorSearch(queryEmbedding, embeddings, k);
      const linearDuration = Date.now() - linearStart;

      console.log(`Linear search: ${linearDuration}ms\n`);

      // Calculate speedup
      const speedup = linearDuration / Math.max(hnswDuration, 1);

      console.log(`=== Results ===`);
      console.log(`Speedup: ${speedup.toFixed(1)}x faster`);
      console.log(`Target: 150x faster`);
      console.log(`Status: ${speedup >= 150 ? '✓ PASSED' : '⚠ BELOW TARGET (still faster)'}\n`);

      // Verify speedup is significant (may not reach 150x in test environment)
      expect(speedup).toBeGreaterThan(10);

      // Log performance metrics
      console.log('Performance Metrics:');
      console.log(`- HNSW: ${hnswDuration}ms`);
      console.log(`- Linear: ${linearDuration}ms`);
      console.log(`- Improvement: ${speedup.toFixed(1)}x`);
      console.log(`- Vectors: ${vectorCount}`);
      console.log(`- Dimensions: 384`);
      console.log(`- K: ${k}\n`);
    }, 120000); // 2 minute timeout

    it('should achieve sub-100ms search latency for 50K vectors', async () => {
      console.log('\n=== Large-Scale Search Latency Test (50K vectors) ===\n');

      const vectorCount = 50000;
      const batchSize = 5000;
      const batches = vectorCount / batchSize;

      console.log(`Generating and inserting ${vectorCount} vectors in ${batches} batches...`);

      for (let batch = 0; batch < batches; batch++) {
        const patterns: QEPattern[] = [];
        const embeddings: number[][] = [];

        for (let i = 0; i < batchSize; i++) {
          const index = batch * batchSize + i;
          patterns.push({
            id: `large-${index}`,
            type: 'test-gen',
            domain: 'test-planning',
            data: { index },
            confidence: 0.8,
            usageCount: 0,
            successCount: 0,
            createdAt: Date.now(),
            lastUsed: Date.now()
          });
          embeddings.push(generateRandomEmbedding(384));
        }

        await agentDB.storeBatch(patterns, embeddings);
        console.log(`Batch ${batch + 1}/${batches} complete`);
      }

      console.log(`\nAll ${vectorCount} vectors inserted\n`);

      // Run multiple search queries
      const queryCount = 10;
      const latencies: number[] = [];

      console.log(`Running ${queryCount} search queries...`);

      for (let i = 0; i < queryCount; i++) {
        const queryEmbedding = generateRandomEmbedding(384);
        const start = Date.now();
        await agentDB.searchSimilar(queryEmbedding, { k: 10 });
        const latency = Date.now() - start;
        latencies.push(latency);

        console.log(`Query ${i + 1}/${queryCount}: ${latency}ms`);
      }

      const avgLatency = latencies.reduce((sum, l) => sum + l, 0) / latencies.length;
      const maxLatency = Math.max(...latencies);
      const minLatency = Math.min(...latencies);

      console.log(`\n=== Results ===`);
      console.log(`Average latency: ${avgLatency.toFixed(2)}ms`);
      console.log(`Min latency: ${minLatency}ms`);
      console.log(`Max latency: ${maxLatency}ms`);
      console.log(`Target: <100ms`);
      console.log(`Status: ${avgLatency < 100 ? '✓ PASSED' : '⚠ ABOVE TARGET'}\n`);

      // Verify reasonable performance
      expect(avgLatency).toBeLessThan(500); // Allow up to 500ms in test environment
    }, 300000); // 5 minute timeout
  });

  describe('Batch Operations Performance', () => {
    it('should achieve <2ms batch insert for 100 patterns (target)', async () => {
      console.log('\n=== Batch Insert Performance Test ===\n');

      const batchSize = 100;
      const iterations = 10;
      const durations: number[] = [];

      console.log(`Running ${iterations} iterations of ${batchSize} pattern inserts...`);

      for (let i = 0; i < iterations; i++) {
        const patterns: QEPattern[] = [];
        const embeddings: number[][] = [];

        for (let j = 0; j < batchSize; j++) {
          patterns.push({
            id: `batch-${i}-${j}`,
            type: 'test-gen',
            domain: 'test-planning',
            data: { batch: i, index: j },
            confidence: 0.8,
            usageCount: 0,
            successCount: 0,
            createdAt: Date.now(),
            lastUsed: Date.now()
          });
          embeddings.push(generateRandomEmbedding(384));
        }

        const start = Date.now();
        await agentDB.storeBatch(patterns, embeddings);
        const duration = Date.now() - start;
        durations.push(duration);

        console.log(`Iteration ${i + 1}/${iterations}: ${duration}ms`);
      }

      const avgDuration = durations.reduce((sum, d) => sum + d, 0) / durations.length;

      console.log(`\n=== Results ===`);
      console.log(`Average: ${avgDuration.toFixed(2)}ms`);
      console.log(`Min: ${Math.min(...durations)}ms`);
      console.log(`Max: ${Math.max(...durations)}ms`);
      console.log(`Target: <2ms`);
      console.log(`Status: ${avgDuration < 2 ? '✓ PASSED' : '⚠ ABOVE TARGET (still fast)'}\n`);

      // Verify reasonable performance (allow more time in test environment)
      expect(avgDuration).toBeLessThan(100);
    }, 60000);
  });

  describe('QUIC Sync Performance', () => {
    it('should achieve <1ms QUIC sync latency (target)', async () => {
      console.log('\n=== QUIC Sync Latency Test ===\n');

      const iterations = 100;
      const latencies: number[] = [];

      console.log(`Running ${iterations} QUIC sync operations...`);

      for (let i = 0; i < iterations; i++) {
        const pattern: QEPattern = {
          id: `quic-${i}`,
          type: 'test-gen',
          domain: 'test-planning',
          data: { index: i },
          confidence: 0.8,
          usageCount: 0,
          successCount: 0,
          createdAt: Date.now(),
          lastUsed: Date.now()
        };

        const embedding = generateRandomEmbedding(384);

        const start = Date.now();
        await agentDB.storePatternWithSync(pattern, embedding);
        const latency = Date.now() - start;
        latencies.push(latency);

        if ((i + 1) % 20 === 0) {
          console.log(`Iteration ${i + 1}/${iterations}: avg ${(latencies.reduce((s, l) => s + l, 0) / latencies.length).toFixed(2)}ms`);
        }
      }

      const avgLatency = latencies.reduce((sum, l) => sum + l, 0) / latencies.length;
      const p95Latency = latencies.sort((a, b) => a - b)[Math.floor(latencies.length * 0.95)];

      console.log(`\n=== Results ===`);
      console.log(`Average latency: ${avgLatency.toFixed(2)}ms`);
      console.log(`P95 latency: ${p95Latency}ms`);
      console.log(`Min latency: ${Math.min(...latencies)}ms`);
      console.log(`Max latency: ${Math.max(...latencies)}ms`);
      console.log(`Target: <1ms`);
      console.log(`Status: ${avgLatency < 1 ? '✓ PASSED' : '⚠ ABOVE TARGET'}\n`);

      // Note: In test environment, QUIC may not achieve <1ms
      // In production with proper QUIC implementation, should hit target
      expect(avgLatency).toBeLessThan(100);
    }, 60000);
  });

  describe('Cache Performance', () => {
    it('should demonstrate cache speedup on repeated queries', async () => {
      console.log('\n=== Cache Performance Test ===\n');

      // Insert some patterns
      const patterns: QEPattern[] = [];
      const embeddings: number[][] = [];

      for (let i = 0; i < 1000; i++) {
        patterns.push({
          id: `cache-${i}`,
          type: 'test-gen',
          domain: 'test-planning',
          data: { index: i },
          confidence: 0.8,
          usageCount: 0,
          successCount: 0,
          createdAt: Date.now(),
          lastUsed: Date.now()
        });
        embeddings.push(generateRandomEmbedding(384));
      }

      await agentDB.storeBatch(patterns, embeddings);

      const queryEmbedding = generateRandomEmbedding(384);

      // First query (cold cache)
      console.log('Running first query (cold cache)...');
      const coldStart = Date.now();
      await agentDB.searchSimilar(queryEmbedding, { k: 10 });
      const coldDuration = Date.now() - coldStart;

      console.log(`Cold cache: ${coldDuration}ms`);

      // Repeated queries (warm cache)
      console.log('\nRunning repeated queries (warm cache)...');
      const warmDurations: number[] = [];

      for (let i = 0; i < 5; i++) {
        const warmStart = Date.now();
        await agentDB.searchSimilar(queryEmbedding, { k: 10 });
        const warmDuration = Date.now() - warmStart;
        warmDurations.push(warmDuration);

        console.log(`Warm cache ${i + 1}: ${warmDuration}ms`);
      }

      const avgWarmDuration = warmDurations.reduce((sum, d) => sum + d, 0) / warmDurations.length;
      const cacheSpeedup = coldDuration / avgWarmDuration;

      console.log(`\n=== Results ===`);
      console.log(`Cold cache: ${coldDuration}ms`);
      console.log(`Warm cache (avg): ${avgWarmDuration.toFixed(2)}ms`);
      console.log(`Cache speedup: ${cacheSpeedup.toFixed(1)}x`);
      console.log(`Status: ${cacheSpeedup > 1 ? '✓ CACHE EFFECTIVE' : '⚠ NO CACHE BENEFIT'}\n`);
    }, 60000);
  });
});

// Helper Functions

// Seeded RNG for deterministic test results
const rng = createSeededRandom(17000);

function generateRandomEmbedding(dimensions: number): number[] {
  const embedding: number[] = [];
  for (let i = 0; i < dimensions; i++) {
    embedding.push(rng.random() * 2 - 1);
  }

  // Normalize to unit vector
  const magnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
  return embedding.map(val => val / magnitude);
}

async function linearVectorSearch(
  query: number[],
  embeddings: number[][],
  k: number
): Promise<Array<{ index: number; similarity: number }>> {
  const results: Array<{ index: number; similarity: number }> = [];

  for (let i = 0; i < embeddings.length; i++) {
    const similarity = cosineSimilarity(query, embeddings[i]);
    results.push({ index: i, similarity });
  }

  results.sort((a, b) => b.similarity - a.similarity);
  return results.slice(0, k);
}

function cosineSimilarity(a: number[], b: number[]): number {
  let dotProduct = 0;
  let magnitudeA = 0;
  let magnitudeB = 0;

  for (let i = 0; i < a.length; i++) {
    dotProduct += a[i] * b[i];
    magnitudeA += a[i] * a[i];
    magnitudeB += b[i] * b[i];
  }

  magnitudeA = Math.sqrt(magnitudeA);
  magnitudeB = Math.sqrt(magnitudeB);

  if (magnitudeA === 0 || magnitudeB === 0) {
    return 0;
  }

  return dotProduct / (magnitudeA * magnitudeB);
}