proffesor-for-testing__agentic-qe/tests/unit/shared/llm/providers/bedrock.test.ts

/**
 * Agentic QE v3 - Bedrock Provider Unit Tests
 * ADR-043: Vendor-Independent LLM Support (Milestone 6)
 *
 * Tests AWS Bedrock provider implementation including:
 * - AWS credential configuration
 * - ARN model ID mapping
 * - Chat completion
 * - Streaming (mocked)
 * - Error handling
 * - Region configuration
 */

import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import {
  BedrockProvider,
  DEFAULT_BEDROCK_CONFIG,
  BEDROCK_MODEL_MAPPING,
  BEDROCK_MODEL_REVERSE_MAPPING,
  type BedrockConfig,
} from '../../../../../src/shared/llm/providers/bedrock';
import { CostTracker } from '../../../../../src/shared/llm/cost-tracker';

// Mock fetch globally
const mockFetch = vi.fn();
global.fetch = mockFetch;

// Mock TokenMetricsCollector to avoid side effects
vi.mock('../../../../../src/learning/token-tracker.js', () => ({
  TokenMetricsCollector: {
    recordTokenUsage: vi.fn(),
  },
}));

describe('BedrockProvider', () => {
  let provider: BedrockProvider;
  const originalEnv = process.env;

  beforeEach(() => {
    // Reset environment
    process.env = { ...originalEnv };
    process.env.AWS_ACCESS_KEY_ID = 'test-access-key-id';
    process.env.AWS_SECRET_ACCESS_KEY = 'test-secret-access-key';
    process.env.AWS_REGION = 'us-east-1';

    // Reset mocks
    mockFetch.mockReset();

    // Create provider instance
    provider = new BedrockProvider();
  });

  afterEach(() => {
    process.env = originalEnv;
  });

  // =========================================================================
  // 1. Constructor and Configuration Tests
  // =========================================================================

  describe('constructor and configuration', () => {
    it('should use default configuration when no config provided', () => {
      const config = provider.getConfig();

      expect(config.model).toBe(DEFAULT_BEDROCK_CONFIG.model);
      expect(config.region).toBe(DEFAULT_BEDROCK_CONFIG.region);
      expect(config.maxTokens).toBe(DEFAULT_BEDROCK_CONFIG.maxTokens);
      expect(config.temperature).toBe(DEFAULT_BEDROCK_CONFIG.temperature);
    });

    it('should override defaults with provided configuration', () => {
      const customConfig: Partial<BedrockConfig> = {
        model: 'anthropic.claude-3-opus-20240229-v1:0',
        region: 'eu-west-1',
        maxTokens: 8192,
        temperature: 0.5,
      };

      const customProvider = new BedrockProvider(customConfig);
      const config = customProvider.getConfig();

      expect(config.model).toBe(customConfig.model);
      expect(config.region).toBe(customConfig.region);
      expect(config.maxTokens).toBe(customConfig.maxTokens);
      expect(config.temperature).toBe(customConfig.temperature);
    });

    it('should have correct provider type and name', () => {
      expect(provider.type).toBe('bedrock');
      expect(provider.name).toBe('AWS Bedrock');
    });
  });

  // =========================================================================
  // 2. AWS Credential Configuration Tests
  // =========================================================================

  describe('AWS credential configuration', () => {
    it('should use environment variables for credentials', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve({
          id: 'msg-123',
          type: 'message',
          role: 'assistant',
          content: [{ type: 'text', text: 'Hello' }],
          model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
          stop_reason: 'end_turn',
          usage: { input_tokens: 10, output_tokens: 5 },
        }),
      });

      const result = await provider.isAvailable();
      expect(result).toBe(true);
    });

    it('should use config credentials over environment variables', () => {
      const customProvider = new BedrockProvider({
        accessKeyId: 'config-access-key',
        secretAccessKey: 'config-secret-key',
        region: 'us-west-2',
      });

      const config = customProvider.getConfig();
      expect(config.accessKeyId).toBe('config-access-key');
      expect(config.secretAccessKey).toBe('config-secret-key');
      expect(config.region).toBe('us-west-2');
    });

    it('should return false when credentials are missing', async () => {
      delete process.env.AWS_ACCESS_KEY_ID;
      delete process.env.AWS_SECRET_ACCESS_KEY;

      const unconfiguredProvider = new BedrockProvider();
      const result = await unconfiguredProvider.isAvailable();

      expect(result).toBe(false);
    });

    it('should support session token for temporary credentials', () => {
      const customProvider = new BedrockProvider({
        accessKeyId: 'access-key',
        secretAccessKey: 'secret-key',
        sessionToken: 'session-token',
      });

      const config = customProvider.getConfig();
      expect(config.sessionToken).toBe('session-token');
    });

    it('should use AWS_SESSION_TOKEN from environment', () => {
      process.env.AWS_SESSION_TOKEN = 'env-session-token';

      const envProvider = new BedrockProvider();
      // The session token is retrieved dynamically during signing
      // Just verify the provider can be created without errors
      expect(envProvider).toBeDefined();
    });
  });

  // =========================================================================
  // 3. Region Configuration Tests
  // =========================================================================

  describe('region configuration', () => {
    it('should use config region first', () => {
      const customProvider = new BedrockProvider({ region: 'ap-southeast-1' });
      const config = customProvider.getConfig();
      expect(config.region).toBe('ap-southeast-1');
    });

    it('should fall back to AWS_REGION environment variable', () => {
      process.env.AWS_REGION = 'eu-central-1';
      const envProvider = new BedrockProvider();
      const config = envProvider.getConfig();
      // Default config has us-east-1, but internal getRegion() would use env
      expect(config.region).toBe('us-east-1'); // From DEFAULT_BEDROCK_CONFIG
    });

    it('should fall back to AWS_DEFAULT_REGION environment variable', () => {
      delete process.env.AWS_REGION;
      process.env.AWS_DEFAULT_REGION = 'eu-north-1';

      // When region is not specified (not explicitly undefined), it uses the default
      const envProvider = new BedrockProvider();
      // Region lookup happens internally during API calls via getRegion()
      // The config stores the default, but getRegion() checks env vars at runtime
      expect(envProvider.getConfig().region).toBe('us-east-1');
    });

    it('should default to us-east-1 when no region specified', () => {
      delete process.env.AWS_REGION;
      delete process.env.AWS_DEFAULT_REGION;

      // Don't pass region at all to use default
      const noRegionProvider = new BedrockProvider();
      expect(noRegionProvider.getConfig().region).toBe('us-east-1');
    });
  });

  // =========================================================================
  // 4. ARN Model ID Mapping Tests
  // =========================================================================

  describe('ARN model ID mapping', () => {
    it('should map claude-3-5-sonnet to Bedrock format', () => {
      const bedrockId = BEDROCK_MODEL_MAPPING['claude-3-5-sonnet-20241022'];
      expect(bedrockId).toBe('anthropic.claude-3-5-sonnet-20241022-v2:0');
    });

    it('should map claude-opus-4-5 to Bedrock format', () => {
      const bedrockId = BEDROCK_MODEL_MAPPING['claude-opus-4-5-20251101'];
      expect(bedrockId).toBe('anthropic.claude-opus-4-5-v1:0');
    });

    it('should map claude-haiku-3-5 to Bedrock format', () => {
      const bedrockId = BEDROCK_MODEL_MAPPING['claude-3-5-haiku-20241022'];
      expect(bedrockId).toBe('anthropic.claude-3-5-haiku-v1:0');
    });

    it('should have reverse mapping for all Bedrock model IDs', () => {
      // Verify all Bedrock model IDs have a reverse mapping
      // Note: Multiple canonical IDs may map to the same Bedrock ID (e.g., claude-opus-4-5 and claude-opus-4-5-20251101)
      // The reverse mapping picks one of them
      const bedrockIds = new Set(Object.values(BEDROCK_MODEL_MAPPING));
      for (const bedrockId of bedrockIds) {
        expect(BEDROCK_MODEL_REVERSE_MAPPING[bedrockId]).toBeDefined();
        // The reverse mapping should point to one of the canonical IDs
        expect(BEDROCK_MODEL_MAPPING[BEDROCK_MODEL_REVERSE_MAPPING[bedrockId]]).toBe(bedrockId);
      }
    });

    it('should support legacy Claude 3 models', () => {
      expect(BEDROCK_MODEL_MAPPING['claude-3-opus-20240229']).toBe('anthropic.claude-3-opus-20240229-v1:0');
      expect(BEDROCK_MODEL_MAPPING['claude-3-sonnet-20240229']).toBe('anthropic.claude-3-sonnet-20240229-v1:0');
      expect(BEDROCK_MODEL_MAPPING['claude-3-haiku-20240307']).toBe('anthropic.claude-3-haiku-20240307-v1:0');
    });

    it('should list all supported models', () => {
      const supportedModels = provider.getSupportedModels();

      expect(supportedModels).toContain('claude-3-5-sonnet-20241022');
      expect(supportedModels).toContain('claude-opus-4-5-20251101');
      expect(supportedModels).toContain('claude-3-5-haiku-20241022');
      expect(supportedModels.length).toBeGreaterThanOrEqual(10);
    });
  });

  // =========================================================================
  // 5. Chat Completion Tests
  // =========================================================================

  describe('chat completion', () => {
    const mockSuccessResponse = {
      id: 'msg-abc123',
      type: 'message',
      role: 'assistant',
      content: [{ type: 'text', text: 'Hello! How can I help you today?' }],
      model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
      stop_reason: 'end_turn',
      usage: { input_tokens: 15, output_tokens: 10 },
    };

    it('should generate response from string input', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve(mockSuccessResponse),
      });

      const response = await provider.generate('Hello, Claude!');

      expect(response.content).toBe('Hello! How can I help you today?');
      expect(response.provider).toBe('bedrock');
      expect(response.usage.promptTokens).toBe(15);
      expect(response.usage.completionTokens).toBe(10);
      expect(response.finishReason).toBe('stop');
    });

    it('should generate response from message array', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve(mockSuccessResponse),
      });

      const messages = [
        { role: 'user' as const, content: 'What is 2+2?' },
      ];

      const response = await provider.generate(messages);

      expect(response.content).toBe('Hello! How can I help you today?');
      expect(response.provider).toBe('bedrock');
    });

    it('should include system prompt in request', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve(mockSuccessResponse),
      });

      await provider.generate('Hello', {
        systemPrompt: 'You are a helpful assistant.',
      });

      // Verify the request body included system prompt
      const callArgs = mockFetch.mock.calls[0];
      const requestBody = JSON.parse(callArgs[1].body);
      expect(requestBody.system).toBe('You are a helpful assistant.');
    });

    it('should respect temperature setting', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve(mockSuccessResponse),
      });

      await provider.generate('Hello', { temperature: 0.3 });

      const callArgs = mockFetch.mock.calls[0];
      const requestBody = JSON.parse(callArgs[1].body);
      expect(requestBody.temperature).toBe(0.3);
    });

    it('should clamp temperature to 0-1 range', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve(mockSuccessResponse),
      });

      await provider.generate('Hello', { temperature: 2.0 });

      const callArgs = mockFetch.mock.calls[0];
      const requestBody = JSON.parse(callArgs[1].body);
      expect(requestBody.temperature).toBe(1);
    });

    it('should include stop sequences when provided', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve(mockSuccessResponse),
      });

      await provider.generate('Hello', { stopSequences: ['STOP', 'END'] });

      const callArgs = mockFetch.mock.calls[0];
      const requestBody = JSON.parse(callArgs[1].body);
      expect(requestBody.stop_sequences).toEqual(['STOP', 'END']);
    });
  });

  // =========================================================================
  // 6. Error Handling Tests
  // =========================================================================

  describe('error handling', () => {
    it('should throw API_KEY_MISSING when credentials not configured', async () => {
      delete process.env.AWS_ACCESS_KEY_ID;
      delete process.env.AWS_SECRET_ACCESS_KEY;

      const unconfiguredProvider = new BedrockProvider();

      await expect(unconfiguredProvider.generate('Hello')).rejects.toMatchObject({
        code: 'API_KEY_MISSING',
        provider: 'bedrock',
      });
    });

    it('should throw API_KEY_INVALID on 401 response', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: false,
        status: 401,
        json: () => Promise.resolve({ message: 'Invalid credentials' }),
      });

      await expect(provider.generate('Hello')).rejects.toMatchObject({
        code: 'API_KEY_INVALID',
        provider: 'bedrock',
      });
    });

    it('should throw API_KEY_INVALID on 403 response', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: false,
        status: 403,
        json: () => Promise.resolve({ message: 'Access denied' }),
      });

      await expect(provider.generate('Hello')).rejects.toMatchObject({
        code: 'API_KEY_INVALID',
        provider: 'bedrock',
      });
    });

    it('should throw RATE_LIMITED on 429 response', async () => {
      // Provider with maxRetries=1 to avoid retry delays in tests
      const noRetryProvider = new BedrockProvider({ maxRetries: 1 });

      mockFetch.mockResolvedValue({
        ok: false,
        status: 429,
        json: () => Promise.resolve({ message: 'Too many requests' }),
      });

      await expect(noRetryProvider.generate('Hello')).rejects.toMatchObject({
        code: 'RATE_LIMITED',
        provider: 'bedrock',
        retryable: true,
      });
    });

    it('should throw CONTEXT_LENGTH_EXCEEDED on validation error', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: false,
        status: 400,
        json: () => Promise.resolve({
          message: 'Input token count exceeds maximum',
          __type: 'ValidationException',
        }),
      });

      await expect(provider.generate('Hello')).rejects.toMatchObject({
        code: 'CONTEXT_LENGTH_EXCEEDED',
        provider: 'bedrock',
        retryable: false,
      });
    });

    it('should throw PROVIDER_UNAVAILABLE on 500 response', async () => {
      // Provider with maxRetries=1 to avoid retry delays in tests
      const noRetryProvider = new BedrockProvider({ maxRetries: 1 });

      mockFetch.mockResolvedValue({
        ok: false,
        status: 500,
        json: () => Promise.resolve({ message: 'Internal server error' }),
      });

      await expect(noRetryProvider.generate('Hello')).rejects.toMatchObject({
        code: 'PROVIDER_UNAVAILABLE',
        provider: 'bedrock',
        retryable: true,
      });
    });

    it('should throw TIMEOUT on request timeout', async () => {
      // Provider with maxRetries=1 to avoid retry delays in tests
      const noRetryProvider = new BedrockProvider({ maxRetries: 1 });

      mockFetch.mockImplementation(() => {
        const error = new Error('Request timed out');
        error.name = 'AbortError';
        return Promise.reject(error);
      });

      await expect(noRetryProvider.generate('Hello', { timeoutMs: 100 })).rejects.toMatchObject({
        code: 'TIMEOUT',
        provider: 'bedrock',
        retryable: true,
      });
    });
  });

  // =========================================================================
  // 7. Health Check Tests
  // =========================================================================

  describe('health check', () => {
    it('should return healthy when API responds successfully', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve({
          id: 'msg-health',
          type: 'message',
          role: 'assistant',
          content: [{ type: 'text', text: 'Hi' }],
          model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
          stop_reason: 'end_turn',
          usage: { input_tokens: 2, output_tokens: 1 },
        }),
      });

      const result = await provider.healthCheck();

      expect(result.healthy).toBe(true);
      expect(result.latencyMs).toBeGreaterThanOrEqual(0);
      expect(result.models).toContain('claude-3-5-sonnet-20241022');
      expect(result.details?.region).toBe('us-east-1');
    });

    it('should return unhealthy when credentials missing', async () => {
      delete process.env.AWS_ACCESS_KEY_ID;
      delete process.env.AWS_SECRET_ACCESS_KEY;

      const unconfiguredProvider = new BedrockProvider();
      const result = await unconfiguredProvider.healthCheck();

      expect(result.healthy).toBe(false);
      expect(result.error).toContain('AWS credentials not configured');
    });

    it('should return unhealthy on API error', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: false,
        status: 500,
        text: () => Promise.resolve('Internal Server Error'),
      });

      const result = await provider.healthCheck();

      expect(result.healthy).toBe(false);
      expect(result.error).toContain('API error: 500');
    });
  });

  // =========================================================================
  // 8. Cost Calculation Tests
  // =========================================================================

  describe('cost calculation', () => {
    it('should calculate cost for Bedrock models', () => {
      const cost = CostTracker.calculateCost('anthropic.claude-3-5-sonnet-20241022-v2:0', {
        promptTokens: 1000,
        completionTokens: 500,
        totalTokens: 1500,
      });

      // $3/1M input, $15/1M output
      expect(cost.inputCost).toBeCloseTo(0.003, 5);
      expect(cost.outputCost).toBeCloseTo(0.0075, 5);
      expect(cost.totalCost).toBeCloseTo(0.0105, 5);
      expect(cost.currency).toBe('USD');
    });

    it('should return cost per token for current model', () => {
      const costPerToken = provider.getCostPerToken();

      expect(costPerToken.input).toBeGreaterThan(0);
      expect(costPerToken.output).toBeGreaterThan(0);
    });

    it('should calculate cost for opus model correctly', () => {
      const cost = CostTracker.calculateCost('anthropic.claude-opus-4-5-v1:0', {
        promptTokens: 1000,
        completionTokens: 500,
        totalTokens: 1500,
      });

      // $15/1M input, $75/1M output
      expect(cost.inputCost).toBeCloseTo(0.015, 5);
      expect(cost.outputCost).toBeCloseTo(0.0375, 5);
      expect(cost.totalCost).toBeCloseTo(0.0525, 5);
    });

    it('should calculate cost for haiku model correctly', () => {
      const cost = CostTracker.calculateCost('anthropic.claude-3-5-haiku-v1:0', {
        promptTokens: 1000,
        completionTokens: 500,
        totalTokens: 1500,
      });

      // $1/1M input, $5/1M output
      expect(cost.inputCost).toBeCloseTo(0.001, 5);
      expect(cost.outputCost).toBeCloseTo(0.0025, 5);
      expect(cost.totalCost).toBeCloseTo(0.0035, 5);
    });
  });

  // =========================================================================
  // 9. Embedding Tests
  // =========================================================================

  describe('embeddings', () => {
    it('should throw MODEL_NOT_FOUND for embed requests', async () => {
      await expect(provider.embed('test text')).rejects.toMatchObject({
        code: 'MODEL_NOT_FOUND',
        provider: 'bedrock',
      });
    });
  });

  // =========================================================================
  // 10. Completion Tests
  // =========================================================================

  describe('completion', () => {
    it('should complete text with code-optimized settings', async () => {
      const mockResponse = {
        id: 'msg-complete',
        type: 'message',
        role: 'assistant',
        content: [{ type: 'text', text: 'function completed() {}' }],
        model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
        stop_reason: 'end_turn',
        usage: { input_tokens: 20, output_tokens: 10 },
      };

      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve(mockResponse),
      });

      const result = await provider.complete('function incomplete() {');

      expect(result.completion).toBe('function completed() {}');
      expect(result.provider).toBe('bedrock');
    });
  });

  // =========================================================================
  // 11. Dispose Tests
  // =========================================================================

  describe('dispose', () => {
    it('should dispose without errors', async () => {
      await expect(provider.dispose()).resolves.toBeUndefined();
    });
  });

  // =========================================================================
  // 12. Default Configuration Tests
  // =========================================================================

  describe('default configuration', () => {
    it('should have correct default model', () => {
      expect(DEFAULT_BEDROCK_CONFIG.model).toBe('anthropic.claude-3-5-sonnet-20241022-v2:0');
    });

    it('should have correct default region', () => {
      expect(DEFAULT_BEDROCK_CONFIG.region).toBe('us-east-1');
    });

    it('should have correct default maxTokens', () => {
      expect(DEFAULT_BEDROCK_CONFIG.maxTokens).toBe(4096);
    });

    it('should have correct default temperature', () => {
      expect(DEFAULT_BEDROCK_CONFIG.temperature).toBe(0.7);
    });

    it('should have correct default timeout', () => {
      expect(DEFAULT_BEDROCK_CONFIG.timeoutMs).toBe(60000);
    });

    it('should enable cache by default', () => {
      expect(DEFAULT_BEDROCK_CONFIG.enableCache).toBe(true);
    });

    it('should enable circuit breaker by default', () => {
      expect(DEFAULT_BEDROCK_CONFIG.enableCircuitBreaker).toBe(true);
    });
  });
});