proffesor-for-testing__agentic-qe/tests/integration/error-recovery.test.js

/**
 * Integration Test Suite: Error Recovery Mechanisms
 * Tests fault tolerance, graceful degradation, and automatic recovery systems
 */

const { spawn } = require('child_process');
const fs = require('fs').promises;
const path = require('path');
const { createSeededRandom } = require('../../src/utils/SeededRandom');

describe('Error Recovery Integration Tests', () => {
  const recoveryNamespace = 'aqe-recovery-test';
  let testWorkspace;

  beforeEach(() => {
    jest.useFakeTimers();
  });

  afterEach(() => {
    jest.useRealTimers();
  });

  beforeAll(async () => {
    testWorkspace = path.join(__dirname, '../../.test-recovery-workspace');
    await fs.mkdir(testWorkspace, { recursive: true });
  });

  afterAll(async () => {
    await fs.rmdir(testWorkspace, { recursive: true, force: true });
  });

  describe('Agent Failure Recovery', () => {
    test('13. Test generator agent failure and recovery', async () => {
      const agentConfig = {
        id: 'test-gen-recovery-001',
        type: 'qe-test-generator',
        task: 'generate-tests-with-recovery',
        maxRetries: 3,
        retryDelay: 1000
      };

      // Store agent configuration
      await execCommand(`npx claude-flow@alpha memory store "agent-config-${agentConfig.id}" '${JSON.stringify(agentConfig)}' --namespace "${recoveryNamespace}"`);

      let attemptCount = 0;
      let lastError = null;

      // Simulate multiple failure attempts
      for (let i = 0; i < agentConfig.maxRetries + 1; i++) {
        try {
          attemptCount++;

          if (i < 2) {
            // Simulate failure for first 2 attempts
            throw new Error(`Agent execution failed - attempt ${attemptCount}`);
          }

          // Success on final attempt
          const result = await simulateAgentRecovery(agentConfig, attemptCount);
          expect(result.status).toBe('completed');
          expect(result.attemptCount).toBe(3);
          break;

        } catch (error) {
          lastError = error;

          // Store failure info for recovery analysis
          const failureInfo = {
            agentId: agentConfig.id,
            attempt: attemptCount,
            error: error.message,
            timestamp: new Date().toISOString(),
            willRetry: i < agentConfig.maxRetries
          };

          await execCommand(`npx claude-flow@alpha memory store "failure-${agentConfig.id}-${attemptCount}" '${JSON.stringify(failureInfo)}' --namespace "${recoveryNamespace}"`);

          if (i < agentConfig.maxRetries) {
            await jest.advanceTimersByTimeAsync(agentConfig.retryDelay);
          }
        }
      }

      expect(attemptCount).toBe(3);
    });

    test('14. Coverage analyzer timeout and fallback', async () => {
      const coverageConfig = {
        id: 'coverage-timeout-001',
        type: 'qe-coverage-analyzer',
        timeout: 2000, // 2 seconds
        fallbackStrategy: 'partial-analysis'
      };

      const startTime = Date.now();

      try {
        // Simulate long-running operation that times out
        const timeoutPromise = new Promise((resolve, reject) => {
          setTimeout(() => {
            reject(new Error('Coverage analysis timeout'));
          }, coverageConfig.timeout + 500);
        });
        await jest.advanceTimersByTimeAsync(coverageConfig.timeout + 500);
        await timeoutPromise;
      } catch (error) {
        const fallbackResult = await simulateFallbackCoverage(coverageConfig);

        expect(fallbackResult.strategy).toBe('partial-analysis');
        expect(fallbackResult.status).toBe('partial-success');
        expect(fallbackResult.coverage).toHaveProperty('estimated');

        const executionTime = Date.now() - startTime;
        expect(executionTime).toBeLessThan(coverageConfig.timeout + 1000);
      }
    });

    test('15. Quality gate circuit breaker pattern', async () => {
      const circuitBreakerConfig = {
        failureThreshold: 3,
        resetTimeout: 5000,
        halfOpenMaxCalls: 2
      };

      let failureCount = 0;
      let circuitState = 'CLOSED'; // CLOSED, OPEN, HALF_OPEN

      // Simulate multiple failures to trigger circuit breaker
      for (let i = 0; i < 5; i++) {
        try {
          if (circuitState === 'OPEN') {
            throw new Error('Circuit breaker is OPEN');
          }

          if (i < circuitBreakerConfig.failureThreshold) {
            failureCount++;
            throw new Error(`Quality gate failure ${failureCount}`);
          }

          // Success after circuit breaker opens and resets
          const result = { status: 'success', attempt: i + 1 };
          expect(result.status).toBe('success');
          break;

        } catch (error) {
          if (failureCount >= circuitBreakerConfig.failureThreshold) {
            circuitState = 'OPEN';

            // Store circuit breaker state
            await execCommand(`npx claude-flow@alpha memory store "circuit-breaker-state" '${JSON.stringify({
              state: circuitState,
              failureCount,
              lastFailure: new Date().toISOString()
            })}' --namespace "${recoveryNamespace}"`);
          }

          expect(error.message).toContain('failure');
        }
      }

      expect(failureCount).toBe(3);
      expect(circuitState).toBe('OPEN');
    });
  });

  describe('System-Level Recovery', () => {
    test('16. Memory corruption recovery', async () => {
      const testData = {
        project: 'test-recovery',
        agents: ['gen', 'exec', 'coverage'],
        status: 'running'
      };

      // Store initial data
      await execCommand(`npx claude-flow@alpha memory store "system-state" '${JSON.stringify(testData)}' --namespace "${recoveryNamespace}"`);

      // Simulate memory corruption by storing invalid data
      try {
        await execCommand(`npx claude-flow@alpha memory store "system-state" 'invalid-json-data' --namespace "${recoveryNamespace}"`);
      } catch (error) {
        // Expected to fail
      }

      // Implement recovery mechanism
      try {
        const corruptedData = await execCommand(`npx claude-flow@alpha memory get "system-state" --namespace "${recoveryNamespace}"`);
        JSON.parse(corruptedData.trim());
      } catch (error) {
        // Data is corrupted, restore from backup
        const backupData = {
          project: 'test-recovery',
          agents: ['gen', 'exec', 'coverage'],
          status: 'recovered',
          recoveryTimestamp: new Date().toISOString()
        };

        await execCommand(`npx claude-flow@alpha memory store "system-state" '${JSON.stringify(backupData)}' --namespace "${recoveryNamespace}"`);

        const recoveredData = await execCommand(`npx claude-flow@alpha memory get "system-state" --namespace "${recoveryNamespace}"`);
        const parsedRecoveredData = JSON.parse(recoveredData.trim());

        expect(parsedRecoveredData.status).toBe('recovered');
        expect(parsedRecoveredData.recoveryTimestamp).toBeDefined();
      }
    });

    test('17. Hook execution failure recovery', async () => {
      const hookConfig = {
        hookId: 'post-test-hook-001',
        type: 'post-task',
        retryPolicy: {
          maxRetries: 2,
          backoffMultiplier: 2,
          initialDelay: 500
        }
      };

      let retryCount = 0;
      const maxRetries = hookConfig.retryPolicy.maxRetries;

      while (retryCount <= maxRetries) {
        try {
          if (retryCount < maxRetries) {
            throw new Error(`Hook execution failed - retry ${retryCount}`);
          }

          // Success on final retry
          const hookResult = {
            hookId: hookConfig.hookId,
            status: 'success',
            retryCount,
            timestamp: new Date().toISOString()
          };

          await execCommand(`npx claude-flow@alpha memory store "hook-result-${hookConfig.hookId}" '${JSON.stringify(hookResult)}' --namespace "${recoveryNamespace}"`);

          expect(hookResult.status).toBe('success');
          expect(hookResult.retryCount).toBe(maxRetries);
          break;

        } catch (error) {
          retryCount++;

          if (retryCount <= maxRetries) {
            const delay = hookConfig.retryPolicy.initialDelay * Math.pow(hookConfig.retryPolicy.backoffMultiplier, retryCount - 1);
            await jest.advanceTimersByTimeAsync(delay);
          } else {
            throw new Error('Hook execution failed after all retries');
          }
        }
      }
    });

    test('18. Cascade failure isolation', async () => {
      const agents = [
        { id: 'agent-1', type: 'qe-test-generator', dependencies: [] },
        { id: 'agent-2', type: 'qe-coverage-analyzer', dependencies: ['agent-1'] },
        { id: 'agent-3', type: 'qe-quality-gate', dependencies: ['agent-1', 'agent-2'] },
        { id: 'agent-4', type: 'qe-performance-tester', dependencies: [] }, // Independent
        { id: 'agent-5', type: 'qe-security-scanner', dependencies: [] } // Independent
      ];

      const isolationResults = [];

      // Simulate agent-1 failure
      const failedAgent = agents[0];
      const failureImpact = calculateFailureImpact(agents, failedAgent.id);

      // Execute independent agents despite failure
      const independentAgents = agents.filter(agent =>
        !failureImpact.affectedAgents.includes(agent.id) && agent.id !== failedAgent.id
      );

      for (const agent of independentAgents) {
        try {
          const result = await simulateIndependentAgentExecution(agent);
          isolationResults.push(result);
        } catch (error) {
          // Should not fail due to isolation
          throw new Error(`Isolation failed: ${agent.id} affected by ${failedAgent.id} failure`);
        }
      }

      expect(isolationResults).toHaveLength(2); // agent-4 and agent-5
      expect(isolationResults.every(result => result.status === 'success')).toBe(true);
      expect(failureImpact.affectedAgents).toEqual(['agent-2', 'agent-3']);
    });
  });

  describe('Data Recovery', () => {
    test('19. Test results recovery after partial execution', async () => {
      const testSuite = {
        id: 'recovery-suite-001',
        totalTests: 100,
        completedTests: 65,
        partialResults: {
          passed: 60,
          failed: 3,
          skipped: 2,
          lastExecutedTest: 'test-065'
        },
        checkpoint: {
          timestamp: new Date().toISOString(),
          resumeFromIndex: 65
        }
      };

      // Store checkpoint data
      await execCommand(`npx claude-flow@alpha memory store "test-suite-checkpoint-${testSuite.id}" '${JSON.stringify(testSuite)}' --namespace "${recoveryNamespace}"`);

      // Simulate system restart and recovery
      const checkpointData = await execCommand(`npx claude-flow@alpha memory get "test-suite-checkpoint-${testSuite.id}" --namespace "${recoveryNamespace}"`);
      const parsedCheckpoint = JSON.parse(checkpointData.trim());

      // Resume execution from checkpoint
      const resumedExecution = {
        ...parsedCheckpoint,
        resumedAt: new Date().toISOString(),
        remainingTests: parsedCheckpoint.totalTests - parsedCheckpoint.completedTests
      };

      expect(resumedExecution.remainingTests).toBe(35);
      expect(resumedExecution.partialResults.lastExecutedTest).toBe('test-065');
      expect(resumedExecution.checkpoint.resumeFromIndex).toBe(65);
    });

    test('20. Artifact recovery and validation', async () => {
      const artifacts = [
        { id: 'test-files-001', type: 'test-suite', path: '/tmp/tests.tar.gz', checksum: 'abc123' },
        { id: 'coverage-report-001', type: 'coverage', path: '/tmp/coverage.json', checksum: 'def456' },
        { id: 'quality-report-001', type: 'quality', path: '/tmp/quality.xml', checksum: 'ghi789' }
      ];

      // Store artifact metadata
      await execCommand(`npx claude-flow@alpha memory store "artifacts-manifest" '${JSON.stringify(artifacts)}' --namespace "${recoveryNamespace}"`);

      // Simulate partial artifact corruption
      const corruptedArtifacts = artifacts.map(artifact => ({
        ...artifact,
        status: artifact.id === 'coverage-report-001' ? 'corrupted' : 'valid'
      }));

      // Recovery process
      const recoveryManifest = [];
      for (const artifact of corruptedArtifacts) {
        if (artifact.status === 'corrupted') {
          // Simulate artifact regeneration
          const recoveredArtifact = {
            ...artifact,
            status: 'recovered',
            originalChecksum: artifact.checksum,
            newChecksum: 'recovered-456',
            recoveryTimestamp: new Date().toISOString()
          };
          recoveryManifest.push(recoveredArtifact);
        } else {
          recoveryManifest.push(artifact);
        }
      }

      const recoveredItem = recoveryManifest.find(item => item.id === 'coverage-report-001');
      expect(recoveredItem.status).toBe('recovered');
      expect(recoveredItem.newChecksum).toBe('recovered-456');
      expect(recoveredItem.recoveryTimestamp).toBeDefined();
    });
  });
});

// Helper functions
async function execCommand(command) {
  return new Promise((resolve, reject) => {
    const [cmd, ...args] = command.split(' ');
    const child = spawn(cmd, args, { stdio: 'pipe' });

    let stdout = '';
    let stderr = '';

    child.stdout.on('data', (data) => stdout += data);
    child.stderr.on('data', (data) => stderr += data);

    child.on('close', (code) => {
      if (code === 0) resolve(stdout);
      else reject(new Error(stderr || stdout));
    });
  });
}

async function simulateAgentRecovery(config, attemptCount) {
  await jest.advanceTimersByTimeAsync(100); // Simulate work

  return {
    agentId: config.id,
    status: 'completed',
    attemptCount,
    timestamp: new Date().toISOString(),
    output: 'recovered-successfully'
  };
}

async function simulateFallbackCoverage(config) {
  await jest.advanceTimersByTimeAsync(500); // Simulate fallback work

  return {
    agentId: config.id,
    strategy: config.fallbackStrategy,
    status: 'partial-success',
    coverage: {
      estimated: true,
      lines: 75, // Estimated coverage
      confidence: 0.8
    },
    timestamp: new Date().toISOString()
  };
}

function calculateFailureImpact(agents, failedAgentId) {
  const affectedAgents = [];

  function findDependents(agentId) {
    for (const agent of agents) {
      if (agent.dependencies.includes(agentId) && !affectedAgents.includes(agent.id)) {
        affectedAgents.push(agent.id);
        findDependents(agent.id); // Recursive dependency check
      }
    }
  }

  findDependents(failedAgentId);

  return {
    failedAgent: failedAgentId,
    affectedAgents,
    isolatedAgents: agents.filter(agent =>
      !affectedAgents.includes(agent.id) && agent.id !== failedAgentId
    ).map(agent => agent.id)
  };
}

const rng = createSeededRandom(27000);

async function simulateIndependentAgentExecution(agent) {
  await jest.advanceTimersByTimeAsync(rng.random() * 500 + 100);

  return {
    agentId: agent.id,
    status: 'success',
    isolated: true,
    timestamp: new Date().toISOString()
  };
}