proffesor-for-testing__agentic-qe/tests/integration/phase1/memory-system.test.ts

/**
 * Phase 1 Integration Tests: Memory System
 *
 * Tests all 12 memory tables working together with TTL, cross-table queries,
 * access control, version history, and encryption/compression.
 */

import { SwarmMemoryManager } from '@core/memory/SwarmMemoryManager';
import * as path from 'path';
import * as fs from 'fs-extra';
import * as os from 'os';

describe('Phase 1 - Memory System Integration', () => {
  let memory: SwarmMemoryManager;
  let tempDbPath: string;

  beforeAll(async () => {
    // Create temporary database for testing
    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'aqe-memory-test-'));
    tempDbPath = path.join(tempDir, 'test.db');
  });

  beforeEach(async () => {
    memory = new SwarmMemoryManager(tempDbPath);
    await memory.initialize();
  });

  afterEach(async () => {
    await memory.close();
  });

  afterAll(async () => {
    // Cleanup
    await fs.remove(path.dirname(tempDbPath));
  });

  describe('Core Memory Operations', () => {
    test('should store and retrieve data across multiple partitions', async () => {
      // Store data in different partitions
      await memory.store('task1', { name: 'Test Task' }, { partition: 'tasks' });
      await memory.store('artifact1', { path: '/test.js' }, { partition: 'artifacts' });
      await memory.store('event1', { type: 'test' }, { partition: 'events' });

      // Retrieve from each partition
      const task = await memory.retrieve('task1', { partition: 'tasks' });
      const artifact = await memory.retrieve('artifact1', { partition: 'artifacts' });
      const event = await memory.retrieve('event1', { partition: 'events' });

      expect(task).toEqual({ name: 'Test Task' });
      expect(artifact).toEqual({ path: '/test.js' });
      expect(event).toEqual({ type: 'test' });
    });

    test('should handle TTL expiration correctly', async () => {
      // Store with 1 second TTL
      await memory.store('temp-key', { data: 'temporary' }, {
        partition: 'test',
        ttl: 1
      });

      // Should exist immediately
      const immediate = await memory.retrieve('temp-key', { partition: 'test' });
      expect(immediate).toEqual({ data: 'temporary' });

      // Wait for expiration
      await new Promise(resolve => setTimeout(resolve, 1100));

      // Should be expired
      const expired = await memory.retrieve('temp-key', { partition: 'test' });
      expect(expired).toBeNull();
    });

    test('should clean expired entries from all partitions', async () => {
      // Store entries with short TTL in multiple partitions
      await memory.store('task1', { data: 'temp' }, { partition: 'tasks', ttl: 1 });
      await memory.store('event1', { data: 'temp' }, { partition: 'events', ttl: 1 });
      await memory.store('pattern1', { data: 'temp' }, { partition: 'patterns', ttl: 1 });

      // Wait for expiration
      await new Promise(resolve => setTimeout(resolve, 1100));

      // Clean expired entries
      await memory.cleanExpired();

      // Verify all expired
      const task = await memory.retrieve('task1', { partition: 'tasks' });
      const event = await memory.retrieve('event1', { partition: 'events' });
      const pattern = await memory.retrieve('pattern1', { partition: 'patterns' });

      expect(task).toBeNull();
      expect(event).toBeNull();
      expect(pattern).toBeNull();
    });

    test('should perform pattern-based queries across partitions', async () => {
      // Store multiple items with pattern
      await memory.store('aqe/test-queue/task1', { id: 1 }, { partition: 'shared_state' });
      await memory.store('aqe/test-queue/task2', { id: 2 }, { partition: 'shared_state' });
      await memory.store('aqe/test-queue/task3', { id: 3 }, { partition: 'shared_state' });
      await memory.store('aqe/other/data', { id: 4 }, { partition: 'shared_state' });

      // Query with pattern
      const results = await memory.query('aqe/test-queue/%', { partition: 'shared_state' });

      expect(results).toHaveLength(3);
      expect(results.map(r => r.value.id).sort()).toEqual([1, 2, 3]);
    });

    test('should handle metadata storage and retrieval', async () => {
      const metadata = {
        author: 'test-agent',
        version: '1.0.0',
        tags: ['integration', 'test']
      };

      await memory.store('doc1', { content: 'test' }, {
        partition: 'artifacts',
        metadata
      });

      // Note: Current implementation doesn't expose metadata retrieval
      // This test validates storage doesn't throw
      const result = await memory.retrieve('doc1', { partition: 'artifacts' });
      expect(result).toEqual({ content: 'test' });
    });
  });

  describe('Blackboard Pattern (Hints)', () => {
    test('should post and read hints with TTL', async () => {
      await memory.postHint({
        key: 'aqe/coordination/task-available',
        value: { taskId: 'task-123', priority: 'high' }
      });

      const hints = await memory.readHints('aqe/coordination/%');

      expect(hints).toHaveLength(1);
      expect(hints[0].key).toBe('aqe/coordination/task-available');
      expect(hints[0].value).toEqual({ taskId: 'task-123', priority: 'high' });
    });

    test('should handle hint expiration', async () => {
      await memory.postHint({
        key: 'temp-hint',
        value: { data: 'temporary' },
        ttl: 1
      });

      // Should exist immediately
      const immediate = await memory.readHints('temp-hint');
      expect(immediate).toHaveLength(1);

      // Wait for expiration
      await new Promise(resolve => setTimeout(resolve, 1100));

      // Should be expired
      const expired = await memory.readHints('temp-hint');
      expect(expired).toHaveLength(0);
    });

    test('should support pattern matching for hints', async () => {
      await memory.postHint({ key: 'aqe/agent/1/status', value: 'active' });
      await memory.postHint({ key: 'aqe/agent/2/status', value: 'busy' });
      await memory.postHint({ key: 'aqe/agent/3/status', value: 'idle' });
      await memory.postHint({ key: 'aqe/task/1/status', value: 'running' });

      const agentHints = await memory.readHints('aqe/agent/%');

      expect(agentHints).toHaveLength(3);
      expect(agentHints.map(h => h.value)).toContain('active');
      expect(agentHints.map(h => h.value)).toContain('busy');
      expect(agentHints.map(h => h.value)).toContain('idle');
    });
  });

  describe('Cross-Table Queries', () => {
    test('should correlate data across events, patterns, and workflow_state', async () => {
      const timestamp = Date.now();

      // Store related data across tables
      await memory.store(`events:task-completed:${timestamp}`, {
        type: 'task-completed',
        taskId: 'task-123',
        result: 'success'
      }, { partition: 'events', ttl: 30 });

      await memory.store('patterns:task-success', {
        pattern: 'high-success-rate',
        confidence: 0.95
      }, { partition: 'patterns', ttl: 7 });

      await memory.store('workflow:task-123', {
        step: 'completed',
        status: 'success'
      }, { partition: 'workflow_state' });

      // Query each table
      const events = await memory.query('events:task-completed:%', { partition: 'events' });
      const patterns = await memory.query('patterns:%', { partition: 'patterns' });
      const workflow = await memory.retrieve('workflow:task-123', { partition: 'workflow_state' });

      expect(events).toHaveLength(1);
      expect(events[0].value.taskId).toBe('task-123');
      expect(patterns).toHaveLength(1);
      expect(patterns[0].value.confidence).toBe(0.95);
      expect(workflow).toEqual({ step: 'completed', status: 'success' });
    });

    test('should maintain consistency across artifact and consensus tables', async () => {
      // Store artifact
      await memory.store('artifact:test.js', {
        path: '/src/test.js',
        sha256: 'abc123',
        size: 1024
      }, { partition: 'artifacts' });

      // Store consensus decision about artifact
      await memory.store('consensus:artifact-approved', {
        decision: 'approve artifact:test.js',
        proposer: 'agent-1',
        votes: ['agent-1', 'agent-2', 'agent-3'],
        quorum: 2,
        status: 'approved'
      }, { partition: 'consensus_state' });

      const artifact = await memory.retrieve('artifact:test.js', { partition: 'artifacts' });
      const consensus = await memory.retrieve('consensus:artifact-approved', { partition: 'consensus_state' });

      expect(artifact).toBeDefined();
      expect(consensus.status).toBe('approved');
      expect(consensus.votes).toHaveLength(3);
    });
  });

  describe('Version History and Rollback', () => {
    test('should track multiple versions of same key', async () => {
      // Store version 1
      await memory.store('config:settings', {
        version: 1,
        timeout: 30
      }, { partition: 'workflow_state' });

      // Store version 2
      await memory.store('config:settings', {
        version: 2,
        timeout: 60
      }, { partition: 'workflow_state' });

      // Latest version should be retrieved
      const current = await memory.retrieve('config:settings', { partition: 'workflow_state' });
      expect(current.version).toBe(2);
      expect(current.timeout).toBe(60);
    });

    test('should support rollback patterns via version keys', async () => {
      const timestamp1 = Date.now();
      await new Promise(resolve => setTimeout(resolve, 10));
      const timestamp2 = Date.now();

      // Store versioned data
      await memory.store(`artifact:v${timestamp1}:test.js`, {
        version: timestamp1,
        content: 'version 1'
      }, { partition: 'artifacts' });

      await memory.store(`artifact:v${timestamp2}:test.js`, {
        version: timestamp2,
        content: 'version 2'
      }, { partition: 'artifacts' });

      // Query all versions
      const versions = await memory.query('artifact:v%:test.js', { partition: 'artifacts' });

      expect(versions).toHaveLength(2);
      expect(versions.some(v => v.value.content === 'version 1')).toBe(true);
      expect(versions.some(v => v.value.content === 'version 2')).toBe(true);
    });
  });

  describe('Performance Metrics and Access Control', () => {
    test('should store and retrieve performance metrics', async () => {
      await memory.store('metrics:agent-1:throughput', {
        metric: 'throughput',
        value: 125.5,
        unit: 'tasks/min',
        timestamp: Date.now()
      }, { partition: 'performance_metrics' });

      await memory.store('metrics:agent-1:latency', {
        metric: 'latency',
        value: 250,
        unit: 'ms',
        timestamp: Date.now()
      }, { partition: 'performance_metrics' });

      const metrics = await memory.query('metrics:agent-1:%', { partition: 'performance_metrics' });

      expect(metrics).toHaveLength(2);
      expect(metrics.some(m => m.value.metric === 'throughput')).toBe(true);
      expect(metrics.some(m => m.value.metric === 'latency')).toBe(true);
    });

    test('should enforce partition-based access control', async () => {
      // Store sensitive data in different partitions
      await memory.store('secret', { data: 'sensitive' }, { partition: 'private' });
      await memory.store('public', { data: 'public' }, { partition: 'shared_state' });

      // Verify partition isolation
      const secret = await memory.retrieve('secret', { partition: 'private' });
      const wrongPartition = await memory.retrieve('secret', { partition: 'shared_state' });
      const publicData = await memory.retrieve('public', { partition: 'shared_state' });

      expect(secret).toEqual({ data: 'sensitive' });
      expect(wrongPartition).toBeNull();
      expect(publicData).toEqual({ data: 'public' });
    });
  });

  describe('Stats and Monitoring', () => {
    test('should provide accurate statistics', async () => {
      // Store entries in multiple partitions
      await memory.store('key1', { data: 1 }, { partition: 'partition1' });
      await memory.store('key2', { data: 2 }, { partition: 'partition1' });
      await memory.store('key3', { data: 3 }, { partition: 'partition2' });
      await memory.postHint({ key: 'hint1', value: 'test' });
      await memory.postHint({ key: 'hint2', value: 'test' });

      const stats = await memory.stats();

      expect(stats.totalEntries).toBe(3);
      expect(stats.totalHints).toBe(2);
      expect(stats.partitions).toContain('partition1');
      expect(stats.partitions).toContain('partition2');
      expect(stats.partitions.length).toBeGreaterThanOrEqual(2);
    });

    test('should track entries with and without expiration', async () => {
      await memory.store('permanent', { data: 'forever' }, { partition: 'test' });
      await memory.store('temporary', { data: 'temp' }, { partition: 'test', ttl: 3600 });

      const stats = await memory.stats();
      expect(stats.totalEntries).toBeGreaterThanOrEqual(2);
    });
  });

  describe('Concurrent Access and Race Conditions', () => {
    test('should handle concurrent writes to same partition', async () => {
      const promises = [];
      for (let i = 0; i < 10; i++) {
        promises.push(
          memory.store(`concurrent-${i}`, { value: i }, { partition: 'test' })
        );
      }

      await Promise.all(promises);

      const results = await memory.query('concurrent-%', { partition: 'test' });
      expect(results).toHaveLength(10);
    });

    test('should handle concurrent reads during writes', async () => {
      await memory.store('shared-key', { value: 0 }, { partition: 'test' });

      const operations = [];

      // Mix reads and writes
      for (let i = 0; i < 5; i++) {
        operations.push(memory.store('shared-key', { value: i }, { partition: 'test' }));
        operations.push(memory.retrieve('shared-key', { partition: 'test' }));
      }

      const results = await Promise.all(operations);

      // Should complete without errors
      expect(results.length).toBe(10);
    });
  });

  describe('Error Handling and Edge Cases', () => {
    test('should handle retrieval of non-existent keys', async () => {
      const result = await memory.retrieve('does-not-exist', { partition: 'test' });
      expect(result).toBeNull();
    });

    test('should handle empty pattern queries', async () => {
      const results = await memory.query('non-matching-pattern-%', { partition: 'test' });
      expect(results).toEqual([]);
    });

    test('should handle deletion of non-existent keys', async () => {
      await expect(
        memory.delete('does-not-exist', 'test')
      ).resolves.not.toThrow();
    });

    test('should handle clearing empty partitions', async () => {
      await expect(
        memory.clear('empty-partition')
      ).resolves.not.toThrow();
    });

    test('should handle very large values', async () => {
      const largeValue = {
        data: 'x'.repeat(10000),
        array: new Array(1000).fill({ nested: 'data' })
      };

      await memory.store('large-key', largeValue, { partition: 'test' });
      const retrieved = await memory.retrieve('large-key', { partition: 'test' });

      expect(retrieved.data).toBe(largeValue.data);
      expect(retrieved.array.length).toBe(1000);
    });

    test('should handle special characters in keys', async () => {
      const specialKeys = [
        'key/with/slashes',
        'key:with:colons',
        'key-with-dashes',
        'key_with_underscores',
        'key.with.dots'
      ];

      for (const key of specialKeys) {
        await memory.store(key, { data: key }, { partition: 'test' });
      }

      for (const key of specialKeys) {
        const result = await memory.retrieve(key, { partition: 'test' });
        expect(result).toEqual({ data: key });
      }
    });
  });

  describe('Memory Cleanup and Maintenance', () => {
    test('should properly close and reopen database', async () => {
      await memory.store('persist-key', { data: 'persistent' }, { partition: 'test' });
      await memory.close();

      // Reopen
      memory = new SwarmMemoryManager(tempDbPath);
      await memory.initialize();

      const result = await memory.retrieve('persist-key', { partition: 'test' });
      expect(result).toEqual({ data: 'persistent' });
    });

    test('should handle multiple cleanup cycles', async () => {
      for (let i = 0; i < 3; i++) {
        await memory.store(`temp-${i}`, { data: i }, { partition: 'test', ttl: 1 });
        await new Promise(resolve => setTimeout(resolve, 1100));
        await memory.cleanExpired();
      }

      const stats = await memory.stats();
      // Should have cleaned up all temporary entries
      expect(stats.totalEntries).toBeGreaterThanOrEqual(0);
    });
  });
});