proffesor-for-testing__agentic-qe/tests/integration/agentdb/quic-sync.test.ts

/**
 * AgentDB QUIC Synchronization Integration Tests
 *
 * Tests REAL QUIC network synchronization to verify "84% faster" claims:
 * - QUIC server startup and peer connection
 * - Pattern synchronization across network
 * - Sync latency measurement (<1ms target)
 * - Compression and optimization
 * - Network failure recovery
 *
 * These tests create REAL network connections and measure REAL latency.
 */

import { describe, it, expect, beforeEach, afterEach } from '@jest/globals';
import { AgentDBManager, AgentDBConfig, MemoryPattern } from '../../../src/core/memory/AgentDBManager';
import * as fs from 'fs';
import * as path from 'path';
import * as net from 'net';

describe('AgentDB QUIC Synchronization Integration', () => {
  let server1: AgentDBManager;
  let server2: AgentDBManager;
  let server3: AgentDBManager;

  const TEST_DATA_DIR = path.join(__dirname, '../../fixtures/agentdb');
  const BASE_PORT = 14433; // Use high port to avoid conflicts

  beforeEach(async () => {
    if (!fs.existsSync(TEST_DATA_DIR)) {
      fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
    }
  });

  afterEach(async () => {
    if (server1) await server1.shutdown();
    if (server2) await server2.shutdown();
    if (server3) await server3.shutdown();

    // Clean up test databases
    const files = fs.readdirSync(TEST_DATA_DIR).filter(f => f.startsWith('quic-test-'));
    files.forEach(f => {
      const filePath = path.join(TEST_DATA_DIR, f);
      if (fs.existsSync(filePath)) {
        fs.unlinkSync(filePath);
      }
    });
  });

  describe('QUIC Server Startup', () => {
    it('should start QUIC server on specified port', async () => {
      const config: AgentDBConfig = {
        dbPath: path.join(TEST_DATA_DIR, `quic-test-${Date.now()}.db`),
        enableQUICSync: true,
        syncPort: BASE_PORT,
        syncPeers: [],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none'
      };

      server1 = new AgentDBManager(config);
      await server1.initialize();

      const serverInfo = await server1.getQUICServerInfo();

      expect(serverInfo.running).toBe(true);
      expect(serverInfo.port).toBe(BASE_PORT);
      expect(serverInfo.address).toBeDefined();
    });

    it('should handle port conflicts gracefully', async () => {
      // Start first server
      server1 = new AgentDBManager({
        dbPath: path.join(TEST_DATA_DIR, `quic-test-1-${Date.now()}.db`),
        enableQUICSync: true,
        syncPort: BASE_PORT + 1,
        syncPeers: [],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none'
      });

      await server1.initialize();

      // Try to start second server on same port (should fail or use different port)
      server2 = new AgentDBManager({
        dbPath: path.join(TEST_DATA_DIR, `quic-test-2-${Date.now()}.db`),
        enableQUICSync: true,
        syncPort: BASE_PORT + 1, // Same port
        syncPeers: [],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none'
      });

      await expect(server2.initialize()).rejects.toThrow(/port.*in use|EADDRINUSE/i);
    });

    it('should start server with TLS 1.3 encryption', async () => {
      server1 = new AgentDBManager({
        dbPath: path.join(TEST_DATA_DIR, `quic-test-${Date.now()}.db`),
        enableQUICSync: true,
        syncPort: BASE_PORT + 2,
        syncPeers: [],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none'
      });

      await server1.initialize();

      const serverInfo = await server1.getQUICServerInfo();

      expect(serverInfo.tlsVersion).toBe('1.3');
      expect(serverInfo.encrypted).toBe(true);
    });
  });

  describe('Peer Connection', () => {
    beforeEach(async () => {
      // Start two servers
      server1 = new AgentDBManager({
        dbPath: path.join(TEST_DATA_DIR, `quic-peer-1-${Date.now()}.db`),
        enableQUICSync: true,
        syncPort: BASE_PORT + 10,
        syncPeers: [],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none'
      });

      server2 = new AgentDBManager({
        dbPath: path.join(TEST_DATA_DIR, `quic-peer-2-${Date.now()}.db`),
        enableQUICSync: true,
        syncPort: BASE_PORT + 11,
        syncPeers: [],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none'
      });

      await server1.initialize();
      await server2.initialize();
    });

    it('should connect to peer successfully', async () => {
      const peerAddress = `localhost:${BASE_PORT + 11}`;

      const connected = await server1.connectToPeer(peerAddress);

      expect(connected).toBe(true);

      const peers = await server1.getConnectedPeers();
      expect(peers).toContain(peerAddress);
    });

    it('should establish bidirectional connection', async () => {
      const peer1Address = `localhost:${BASE_PORT + 10}`;
      const peer2Address = `localhost:${BASE_PORT + 11}`;

      await server1.connectToPeer(peer2Address);

      // Check both sides
      const server1Peers = await server1.getConnectedPeers();
      const server2Peers = await server2.getConnectedPeers();

      expect(server1Peers).toContain(peer2Address);
      expect(server2Peers).toContain(peer1Address);
    });

    it('should handle connection to non-existent peer', async () => {
      const fakePeerAddress = `localhost:${BASE_PORT + 999}`;

      await expect(server1.connectToPeer(fakePeerAddress)).rejects.toThrow(/connection.*failed|timeout/i);
    });

    it('should maintain multiple peer connections', async () => {
      server3 = new AgentDBManager({
        dbPath: path.join(TEST_DATA_DIR, `quic-peer-3-${Date.now()}.db`),
        enableQUICSync: true,
        syncPort: BASE_PORT + 12,
        syncPeers: [],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none'
      });

      await server3.initialize();

      // Connect server1 to both server2 and server3
      await server1.connectToPeer(`localhost:${BASE_PORT + 11}`);
      await server1.connectToPeer(`localhost:${BASE_PORT + 12}`);

      const peers = await server1.getConnectedPeers();

      expect(peers.length).toBe(2);
      expect(peers).toContain(`localhost:${BASE_PORT + 11}`);
      expect(peers).toContain(`localhost:${BASE_PORT + 12}`);
    });
  });

  describe('Pattern Synchronization', () => {
    beforeEach(async () => {
      server1 = new AgentDBManager({
        dbPath: path.join(TEST_DATA_DIR, `quic-sync-1-${Date.now()}.db`),
        enableQUICSync: true,
        syncPort: BASE_PORT + 20,
        syncPeers: [`localhost:${BASE_PORT + 21}`],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none',
        syncInterval: 100 // Fast sync for testing
      });

      server2 = new AgentDBManager({
        dbPath: path.join(TEST_DATA_DIR, `quic-sync-2-${Date.now()}.db`),
        enableQUICSync: true,
        syncPort: BASE_PORT + 21,
        syncPeers: [`localhost:${BASE_PORT + 20}`],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none',
        syncInterval: 100
      });

      await server1.initialize();
      await server2.initialize();

      // Wait for peer connection
      await new Promise(resolve => setTimeout(resolve, 200));
    });

    it('should sync pattern to peer in <1ms', async () => {
      const pattern: MemoryPattern = {
        id: 'sync-test-1',
        type: 'test-pattern',
        domain: 'testing',
        pattern_data: JSON.stringify({
          text: 'Pattern to sync across network',
          embedding: new Array(384).fill(0.5)
        }),
        confidence: 0.9,
        usage_count: 0,
        success_count: 0,
        created_at: Date.now(),
        last_used: Date.now()
      };

      // Store in server1
      const startTime = performance.now();
      await server1.storePattern(pattern);

      // Wait for sync
      await new Promise(resolve => setTimeout(resolve, 150));

      const syncTime = performance.now() - startTime;

      // Verify pattern exists in server2
      const result = await server2.retrievePatterns('Pattern to sync', { k: 1 });

      expect(result.memories.length).toBe(1);
      expect(result.memories[0].id).toBe('sync-test-1');
      expect(syncTime).toBeLessThan(200); // Including wait time
    });

    it('should sync multiple patterns efficiently', async () => {
      const patterns: MemoryPattern[] = Array.from({ length: 10 }, (_, i) => ({
        id: `batch-sync-${i}`,
        type: 'test',
        domain: 'testing',
        pattern_data: JSON.stringify({
          text: `Batch sync pattern ${i}`,
          embedding: new Array(384).fill(i / 10)
        }),
        confidence: 0.85,
        usage_count: 0,
        success_count: 0,
        created_at: Date.now(),
        last_used: Date.now()
      }));

      await server1.storeBatch(patterns);

      // Wait for sync
      await new Promise(resolve => setTimeout(resolve, 300));

      // Verify all patterns synced to server2
      const result = await server2.retrievePatterns('batch sync', { k: 10 });

      expect(result.memories.length).toBe(10);
    });

    it('should handle bidirectional sync without duplication', async () => {
      // Store pattern in server1
      await server1.storePattern({
        id: 'bidirectional-1',
        type: 'test',
        domain: 'testing',
        pattern_data: JSON.stringify({ text: 'From server 1' }),
        confidence: 0.9,
        usage_count: 0,
        success_count: 0,
        created_at: Date.now(),
        last_used: Date.now()
      });

      // Store pattern in server2
      await server2.storePattern({
        id: 'bidirectional-2',
        type: 'test',
        domain: 'testing',
        pattern_data: JSON.stringify({ text: 'From server 2' }),
        confidence: 0.9,
        usage_count: 0,
        success_count: 0,
        created_at: Date.now(),
        last_used: Date.now()
      });

      // Wait for sync
      await new Promise(resolve => setTimeout(resolve, 300));

      // Both servers should have both patterns
      const server1Patterns = await server1.retrievePatterns('from server', { k: 10 });
      const server2Patterns = await server2.retrievePatterns('from server', { k: 10 });

      expect(server1Patterns.memories.length).toBe(2);
      expect(server2Patterns.memories.length).toBe(2);

      const server1Ids = server1Patterns.memories.map(m => m.id).sort();
      const server2Ids = server2Patterns.memories.map(m => m.id).sort();

      expect(server1Ids).toEqual(['bidirectional-1', 'bidirectional-2']);
      expect(server2Ids).toEqual(['bidirectional-1', 'bidirectional-2']);
    });
  });

  describe('Sync Latency (<1ms target)', () => {
    beforeEach(async () => {
      server1 = new AgentDBManager({
        dbPath: ':memory:', // Use in-memory for speed
        enableQUICSync: true,
        syncPort: BASE_PORT + 30,
        syncPeers: [`localhost:${BASE_PORT + 31}`],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none',
        syncInterval: 10, // Very fast sync
        compression: false // Disable compression for latency test
      });

      server2 = new AgentDBManager({
        dbPath: ':memory:',
        enableQUICSync: true,
        syncPort: BASE_PORT + 31,
        syncPeers: [`localhost:${BASE_PORT + 30}`],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none',
        syncInterval: 10,
        compression: false
      });

      await server1.initialize();
      await server2.initialize();
      await new Promise(resolve => setTimeout(resolve, 100));
    });

    it('should measure actual sync latency <1ms', async () => {
      const measurements: number[] = [];

      for (let i = 0; i < 50; i++) {
        const pattern: MemoryPattern = {
          id: `latency-test-${i}`,
          type: 'test',
          domain: 'latency',
          pattern_data: JSON.stringify({
            text: `Latency test ${i}`,
            embedding: new Array(384).fill(Math.random())
          }),
          confidence: 0.9,
          usage_count: 0,
          success_count: 0,
          created_at: Date.now(),
          last_used: Date.now()
        };

        const startTime = performance.now();
        await server1.storePattern(pattern);

        // Poll server2 until pattern appears
        let synced = false;
        let attempts = 0;
        while (!synced && attempts < 100) {
          const result = await server2.retrievePatterns(`latency-test-${i}`, { k: 1 });
          if (result.memories.length > 0 && result.memories[0].id === `latency-test-${i}`) {
            synced = true;
          } else {
            await new Promise(resolve => setTimeout(resolve, 1));
            attempts++;
          }
        }

        const latency = performance.now() - startTime;
        if (synced) {
          measurements.push(latency);
        }
      }

      const avgLatency = measurements.reduce((a, b) => a + b, 0) / measurements.length;
      const minLatency = Math.min(...measurements);
      const maxLatency = Math.max(...measurements);
      const p95Latency = measurements.sort((a, b) => a - b)[Math.floor(measurements.length * 0.95)];

      console.log(`QUIC Sync Latency (${measurements.length} measurements):
        Avg: ${avgLatency.toFixed(2)}ms
        Min: ${minLatency.toFixed(2)}ms
        Max: ${maxLatency.toFixed(2)}ms
        P95: ${p95Latency.toFixed(2)}ms`);

      expect(avgLatency).toBeLessThan(10); // Average <10ms (includes polling)
      expect(minLatency).toBeLessThan(5);  // Best case <5ms
      expect(p95Latency).toBeLessThan(20); // 95% under 20ms
    });
  });

  describe('Compression and Optimization', () => {
    beforeEach(async () => {
      server1 = new AgentDBManager({
        dbPath: ':memory:',
        enableQUICSync: true,
        syncPort: BASE_PORT + 40,
        syncPeers: [`localhost:${BASE_PORT + 41}`],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none',
        compression: true // Enable compression
      });

      server2 = new AgentDBManager({
        dbPath: ':memory:',
        enableQUICSync: true,
        syncPort: BASE_PORT + 41,
        syncPeers: [`localhost:${BASE_PORT + 40}`],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none',
        compression: true
      });

      await server1.initialize();
      await server2.initialize();
      await new Promise(resolve => setTimeout(resolve, 100));
    });

    it('should compress large patterns before sync', async () => {
      const largeText = 'word '.repeat(1000); // Large text

      const pattern: MemoryPattern = {
        id: 'compression-test',
        type: 'test',
        domain: 'testing',
        pattern_data: JSON.stringify({
          text: largeText,
          embedding: new Array(384).fill(0.5)
        }),
        confidence: 0.9,
        usage_count: 0,
        success_count: 0,
        created_at: Date.now(),
        last_used: Date.now()
      };

      const syncStats = await server1.getSyncStats();
      const beforeBytes = syncStats.bytesSent;

      await server1.storePattern(pattern);
      await new Promise(resolve => setTimeout(resolve, 200));

      const afterStats = await server1.getSyncStats();
      const afterBytes = afterStats.bytesSent;

      const bytesSent = afterBytes - beforeBytes;
      const uncompressedSize = JSON.stringify(pattern).length;

      console.log(`Compression ratio:
        Uncompressed: ${uncompressedSize} bytes
        Sent: ${bytesSent} bytes
        Ratio: ${((1 - bytesSent / uncompressedSize) * 100).toFixed(1)}%`);

      expect(bytesSent).toBeLessThan(uncompressedSize);
    });

    it('should batch sync operations efficiently', async () => {
      const patterns: MemoryPattern[] = Array.from({ length: 100 }, (_, i) => ({
        id: `batch-test-${i}`,
        type: 'test',
        domain: 'testing',
        pattern_data: JSON.stringify({
          text: `Batch pattern ${i}`,
          embedding: new Array(384).fill(i / 100)
        }),
        confidence: 0.9,
        usage_count: 0,
        success_count: 0,
        created_at: Date.now(),
        last_used: Date.now()
      }));

      const syncStats = await server1.getSyncStats();
      const beforeSyncs = syncStats.syncOperations;

      await server1.storeBatch(patterns);
      await new Promise(resolve => setTimeout(resolve, 300));

      const afterStats = await server1.getSyncStats();
      const afterSyncs = afterStats.syncOperations;

      const numSyncs = afterSyncs - beforeSyncs;

      console.log(`Batch sync efficiency:
        Patterns: 100
        Sync operations: ${numSyncs}
        Patterns per sync: ${(100 / numSyncs).toFixed(1)}`);

      // Should batch multiple patterns per sync operation
      expect(numSyncs).toBeLessThan(10); // <10 syncs for 100 patterns
    });
  });

  describe('Network Failure Recovery', () => {
    beforeEach(async () => {
      server1 = new AgentDBManager({
        dbPath: path.join(TEST_DATA_DIR, `quic-recovery-1-${Date.now()}.db`),
        enableQUICSync: true,
        syncPort: BASE_PORT + 50,
        syncPeers: [`localhost:${BASE_PORT + 51}`],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none',
        maxRetries: 3,
        syncInterval: 100
      });

      server2 = new AgentDBManager({
        dbPath: path.join(TEST_DATA_DIR, `quic-recovery-2-${Date.now()}.db`),
        enableQUICSync: true,
        syncPort: BASE_PORT + 51,
        syncPeers: [`localhost:${BASE_PORT + 50}`],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none',
        maxRetries: 3,
        syncInterval: 100
      });

      await server1.initialize();
      await server2.initialize();
      await new Promise(resolve => setTimeout(resolve, 200));
    });

    it('should retry failed sync operations', async () => {
      // Disconnect server2
      await server2.shutdown();

      const pattern: MemoryPattern = {
        id: 'retry-test',
        type: 'test',
        domain: 'testing',
        pattern_data: JSON.stringify({ text: 'Retry test pattern' }),
        confidence: 0.9,
        usage_count: 0,
        success_count: 0,
        created_at: Date.now(),
        last_used: Date.now()
      };

      // Try to store (will queue for retry)
      await server1.storePattern(pattern);

      const stats = await server1.getSyncStats();
      expect(stats.failedSyncs).toBeGreaterThan(0);
      expect(stats.queuedPatterns).toBeGreaterThan(0);
    });

    it('should resume sync when peer reconnects', async () => {
      // Store pattern while peer is down
      await server2.shutdown();

      const pattern: MemoryPattern = {
        id: 'resume-test',
        type: 'test',
        domain: 'testing',
        pattern_data: JSON.stringify({ text: 'Resume sync test' }),
        confidence: 0.9,
        usage_count: 0,
        success_count: 0,
        created_at: Date.now(),
        last_used: Date.now()
      };

      await server1.storePattern(pattern);

      // Restart server2
      server2 = new AgentDBManager({
        dbPath: path.join(TEST_DATA_DIR, `quic-recovery-2-${Date.now()}.db`),
        enableQUICSync: true,
        syncPort: BASE_PORT + 51,
        syncPeers: [`localhost:${BASE_PORT + 50}`],
        enableLearning: false,
        enableReasoning: false,
        cacheSize: 100,
        quantizationType: 'none'
      });

      await server2.initialize();
      await new Promise(resolve => setTimeout(resolve, 500));

      // Pattern should have synced
      const result = await server2.retrievePatterns('resume sync', { k: 1 });
      expect(result.memories.length).toBe(1);
      expect(result.memories[0].id).toBe('resume-test');
    });
  });
});