proffesor-for-testing__agentic-qe/tests/unit/cli/commands/llm-router.test.ts

/**
 * LLM Router CLI Commands Tests
 * ADR-043: Vendor-Independent LLM Support - Milestone 10
 */

import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { createLLMRouterCommand } from '../../../../src/cli/commands/llm-router.js';
import type { Command } from 'commander';

// Mock console.log and console.error for output testing
let mockConsoleLog: ReturnType<typeof vi.spyOn>;
let mockConsoleError: ReturnType<typeof vi.spyOn>;
let mockExit: ReturnType<typeof vi.spyOn>;

// Helper to parse command and execute it
async function executeCommand(
  cmd: Command,
  args: string[],
): Promise<void> {
  // Configure commander to not exit on error
  cmd.exitOverride();
  cmd.configureOutput({
    writeOut: () => {},
    writeErr: () => {},
  });

  // Parse will trigger the action - use 'node' as default (first two items are node and script)
  await cmd.parseAsync(args, { from: 'user' });
}

describe('LLM Router CLI Commands', () => {
  let command: ReturnType<typeof createLLMRouterCommand>;
  let tmpConfigRoot: string;
  let savedConfigRootEnv: string | undefined;

  beforeEach(() => {
    // Redirect the CLI's config persistence to a per-test temp dir so
    // `aqe llm config --set` doesn't mutate the real .agentic-qe/.
    tmpConfigRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'aqe-cli-llm-'));
    savedConfigRootEnv = process.env.AQE_CONFIG_ROOT;
    process.env.AQE_CONFIG_ROOT = tmpConfigRoot;

    command = createLLMRouterCommand();
    mockConsoleLog = vi.spyOn(console, 'log').mockImplementation(() => {});
    mockConsoleError = vi.spyOn(console, 'error').mockImplementation(() => {});
    mockExit = vi.spyOn(process, 'exit').mockImplementation((() => {}) as never);
  });

  afterEach(() => {
    vi.restoreAllMocks();
    if (savedConfigRootEnv === undefined) {
      delete process.env.AQE_CONFIG_ROOT;
    } else {
      process.env.AQE_CONFIG_ROOT = savedConfigRootEnv;
    }
    if (fs.existsSync(tmpConfigRoot)) {
      fs.rmSync(tmpConfigRoot, { recursive: true, force: true });
    }
  });

  describe('createLLMRouterCommand', () => {
    it('should create a command with name "llm"', () => {
      expect(command.name()).toBe('llm');
    });

    it('should have a description', () => {
      expect(command.description()).toBeTruthy();
      expect(command.description()).toContain('LLM Router');
    });

    it('should have subcommands', () => {
      const subcommandNames = command.commands.map(c => c.name());
      expect(subcommandNames).toContain('providers');
      expect(subcommandNames).toContain('models');
      expect(subcommandNames).toContain('route');
      expect(subcommandNames).toContain('config');
      expect(subcommandNames).toContain('health');
      expect(subcommandNames).toContain('cost');
    });
  });

  describe('providers subcommand', () => {
    it('should exist and have correct options', () => {
      const providersCmd = command.commands.find(c => c.name() === 'providers');
      expect(providersCmd).toBeDefined();

      const options = providersCmd!.options.map(o => o.long);
      expect(options).toContain('--json');
      expect(options).toContain('--verbose');
    });

    it('should list all provider types via parse', async () => {
      await executeCommand(command, ['providers', '--json']);

      expect(mockConsoleLog).toHaveBeenCalled();

      // Get the JSON output
      const jsonOutput = mockConsoleLog.mock.calls[0][0];
      const providers = JSON.parse(jsonOutput);

      expect(Array.isArray(providers)).toBe(true);
      expect(providers.length).toBeGreaterThan(0);

      // Check for expected providers
      const providerNames = providers.map((p: { provider: string }) => p.provider);
      expect(providerNames).toContain('claude');
      expect(providerNames).toContain('openai');
      expect(providerNames).toContain('ollama');
    });
  });

  describe('models subcommand', () => {
    it('should exist and have correct options', () => {
      const modelsCmd = command.commands.find(c => c.name() === 'models');
      expect(modelsCmd).toBeDefined();

      const options = modelsCmd!.options.map(o => o.long);
      expect(options).toContain('--provider');
      expect(options).toContain('--tier');
      expect(options).toContain('--json');
    });

    it('should list models as JSON', async () => {
      await executeCommand(command, ['models', '--json']);

      expect(mockConsoleLog).toHaveBeenCalled();

      const jsonOutput = mockConsoleLog.mock.calls[0][0];
      const models = JSON.parse(jsonOutput);

      expect(Array.isArray(models)).toBe(true);
      expect(models.length).toBeGreaterThan(0);

      // Check model structure
      const firstModel = models[0];
      expect(firstModel).toHaveProperty('canonicalId');
      expect(firstModel).toHaveProperty('tier');
      expect(firstModel).toHaveProperty('family');
    });

    it('should filter models by provider', async () => {
      await executeCommand(command, ['models', '--json', '--provider', 'claude']);

      const jsonOutput = mockConsoleLog.mock.calls[0][0];
      const models = JSON.parse(jsonOutput);

      expect(models.length).toBeGreaterThan(0);
      // All returned models should have Claude provider ID
      for (const model of models) {
        expect(model.providerIds).toHaveProperty('claude');
      }
    });

    it('should filter models by tier', async () => {
      await executeCommand(command, ['models', '--json', '--tier', 'flagship']);

      const jsonOutput = mockConsoleLog.mock.calls[0][0];
      const models = JSON.parse(jsonOutput);

      expect(models.length).toBeGreaterThan(0);
      // All returned models should be flagship tier
      for (const model of models) {
        expect(model.tier).toBe('flagship');
      }
    });
  });

  describe('route subcommand', () => {
    it('should exist and have correct arguments and options', () => {
      const routeCmd = command.commands.find(c => c.name() === 'route');
      expect(routeCmd).toBeDefined();

      // Check for task argument
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
      expect((routeCmd as any).registeredArguments.length).toBe(1);
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
      expect((routeCmd as any).registeredArguments[0].name()).toBe('task');

      const options = routeCmd!.options.map(o => o.long);
      expect(options).toContain('--agent');
      expect(options).toContain('--complexity');
      expect(options).toContain('--mode');
      expect(options).toContain('--json');
    });

    it('should route a security task to Claude', async () => {
      await executeCommand(command, ['route', 'security vulnerability scan', '--json']);

      const jsonOutput = mockConsoleLog.mock.calls[0][0];
      const decision = JSON.parse(jsonOutput);

      expect(decision).toHaveProperty('provider');
      expect(decision).toHaveProperty('model');
      expect(decision).toHaveProperty('reason');
      expect(decision).toHaveProperty('confidence');
      expect(decision.provider).toBe('claude');
    });

    it('should route with agent type consideration', async () => {
      await executeCommand(command, ['route', 'analyze code', '--json', '--agent', 'security-auditor']);

      const jsonOutput = mockConsoleLog.mock.calls[0][0];
      const decision = JSON.parse(jsonOutput);

      expect(decision.provider).toBe('claude');
      expect(decision.reason).toContain('Security');
    });

    it('should route simple tasks to efficient models', async () => {
      await executeCommand(command, [
        'route',
        'simple formatting task',
        '--json',
        '--complexity',
        'trivial',
        '--mode',
        'cost-optimized',
      ]);

      const jsonOutput = mockConsoleLog.mock.calls[0][0];
      const decision = JSON.parse(jsonOutput);

      // Should select cost-effective model
      expect(decision.model).toContain('mini');
    });
  });

  describe('config subcommand', () => {
    it('should exist and have correct options', () => {
      const configCmd = command.commands.find(c => c.name() === 'config');
      expect(configCmd).toBeDefined();

      const options = configCmd!.options.map(o => o.long);
      expect(options).toContain('--set');
      expect(options).toContain('--json');
    });

    it('should output configuration as JSON', async () => {
      await executeCommand(command, ['config', '--json']);

      const jsonOutput = mockConsoleLog.mock.calls[0][0];
      const config = JSON.parse(jsonOutput);

      expect(config).toHaveProperty('mode');
      expect(config).toHaveProperty('defaultProvider');
      expect(config).toHaveProperty('defaultModel');
      expect(config).toHaveProperty('enableMetrics');
    });

    it('should set and persist configuration value', async () => {
      // Set mode to cost-optimized
      await executeCommand(command, ['config', '--set', 'mode=cost-optimized']);

      expect(mockConsoleLog).toHaveBeenCalled();

      // Verify the setting was applied
      mockConsoleLog.mockClear();
      // Create new command instance to verify persistence (in-memory for tests)
      const newCommand = createLLMRouterCommand();
      await executeCommand(newCommand, ['config', '--json']);

      const jsonOutput = mockConsoleLog.mock.calls[0][0];
      const config = JSON.parse(jsonOutput);
      expect(config.mode).toBe('cost-optimized');
    });

    it('writes the setting through to disk so a fresh process picks it up', async () => {
      // Simulate the cross-process case: --set writes the file.
      await executeCommand(command, ['config', '--set', 'defaultProvider=gemini']);

      const configPath = path.join(tmpConfigRoot, '.agentic-qe', 'llm-config.json');
      expect(fs.existsSync(configPath)).toBe(true);
      const onDisk = JSON.parse(fs.readFileSync(configPath, 'utf8'));
      expect(onDisk.defaultProvider).toBe('gemini');
    });

    it('strips secrets defensively on persist (apiKey never reaches disk)', async () => {
      // The config-store strips any apiKey fields it sees in the
      // serialized payload — this protects against a future bug that
      // ever copies an env API key into the in-memory config.
      await executeCommand(command, ['config', '--set', 'defaultProvider=openai']);

      const configPath = path.join(tmpConfigRoot, '.agentic-qe', 'llm-config.json');
      const raw = fs.readFileSync(configPath, 'utf8');
      expect(raw).not.toMatch(/sk-/);
      expect(raw).not.toMatch(/apiKey/);
    });

    it('refuses to save apiKey with a clear error pointing to env vars', async () => {
      // Audit Fix #9: previously `--set apiKey=...` returned "Unknown
      // config key" which was unhelpful. Now it throws a clear error
      // that explains api keys go in env vars.
      let thrown: Error | undefined;
      try {
        await executeCommand(command, ['config', '--set', 'apiKey=sk-fake-123']);
      } catch (e) {
        thrown = e as Error;
      }
      expect(thrown).toBeDefined();
      expect(thrown!.message).toMatch(/environment variable|ANTHROPIC_API_KEY|GEMINI_API_KEY/i);

      // And nothing was written to disk
      const configPath = path.join(tmpConfigRoot, '.agentic-qe', 'llm-config.json');
      if (fs.existsSync(configPath)) {
        const raw = fs.readFileSync(configPath, 'utf8');
        expect(raw).not.toMatch(/sk-fake-123/);
      }
    });
  });

  describe('health subcommand', () => {
    it('should exist and have correct options', () => {
      const healthCmd = command.commands.find(c => c.name() === 'health');
      expect(healthCmd).toBeDefined();

      const options = healthCmd!.options.map(o => o.long);
      expect(options).toContain('--json');
      expect(options).toContain('--timeout');
    });

    it('should return health status for all providers', async () => {
      await executeCommand(command, ['health', '--json']);

      const jsonOutput = mockConsoleLog.mock.calls[0][0];
      const healthResults = JSON.parse(jsonOutput);

      expect(Array.isArray(healthResults)).toBe(true);
      expect(healthResults.length).toBeGreaterThan(0);

      // Check structure of health result
      const firstResult = healthResults[0];
      expect(firstResult).toHaveProperty('provider');
      expect(firstResult).toHaveProperty('healthy');
    });
  });

  describe('cost subcommand', () => {
    it('should exist and have correct arguments and options', () => {
      const costCmd = command.commands.find(c => c.name() === 'cost');
      expect(costCmd).toBeDefined();

      // Check for model argument
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
      expect((costCmd as any).registeredArguments.length).toBe(1);
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
      expect((costCmd as any).registeredArguments[0].name()).toBe('model');

      const options = costCmd!.options.map(o => o.long);
      expect(options).toContain('--tokens');
      expect(options).toContain('--json');
    });

    it('should estimate cost for a known model', async () => {
      await executeCommand(command, ['cost', 'claude-sonnet-4-6', '--json', '--tokens', '10000']);

      const jsonOutput = mockConsoleLog.mock.calls[0][0];
      const costEstimate = JSON.parse(jsonOutput);

      expect(costEstimate).toHaveProperty('model');
      expect(costEstimate).toHaveProperty('tokens');
      expect(costEstimate).toHaveProperty('inputCostPer1M');
      expect(costEstimate).toHaveProperty('outputCostPer1M');
      expect(costEstimate).toHaveProperty('estimatedTotalCost');
      expect(costEstimate.tokens).toBe(10000);
    });

    it('should error for unknown model', async () => {
      // Unknown model triggers console.error and process.exit
      await executeCommand(command, ['cost', 'unknown-model-xyz', '--json']);

      // Expected - console.error was called
      expect(mockConsoleError).toHaveBeenCalled();
    });
  });

  describe('command help', () => {
    it('should have help text with subcommands listed', () => {
      const helpText = command.helpInformation();

      expect(helpText).toContain('providers');
      expect(helpText).toContain('models');
      expect(helpText).toContain('route');
      expect(helpText).toContain('config');
      expect(helpText).toContain('health');
      expect(helpText).toContain('cost');
    });
  });
});

describe('LLM Router Integration', () => {
  beforeEach(() => {
    mockConsoleLog = vi.spyOn(console, 'log').mockImplementation(() => {});
    mockConsoleError = vi.spyOn(console, 'error').mockImplementation(() => {});
    mockExit = vi.spyOn(process, 'exit').mockImplementation((() => {}) as never);
  });

  afterEach(() => {
    vi.restoreAllMocks();
  });

  it('should support all 8 provider types', async () => {
    const expectedProviders = [
      'claude',
      'openai',
      'ollama',
      'openrouter',
      'gemini',
      'azure-openai',
      'bedrock',
      'onnx',
    ];

    const command = createLLMRouterCommand();
    await executeCommand(command, ['providers', '--json']);

    const jsonOutput = mockConsoleLog.mock.calls[0][0];
    const providers = JSON.parse(jsonOutput);

    const providerNames = providers.map((p: { provider: string }) => p.provider);
    for (const expected of expectedProviders) {
      expect(providerNames).toContain(expected);
    }
  });

  it('should support 4 routing modes', async () => {
    const modes = ['manual', 'rule-based', 'cost-optimized', 'performance-optimized'];

    for (const mode of modes) {
      mockConsoleLog.mockClear();
      const command = createLLMRouterCommand();
      await executeCommand(command, ['route', 'test task', '--json', '--mode', mode]);

      expect(mockConsoleLog).toHaveBeenCalled();
      const jsonOutput = mockConsoleLog.mock.calls[0][0];
      const decision = JSON.parse(jsonOutput);

      expect(decision).toHaveProperty('provider');
      expect(decision).toHaveProperty('model');
    }
  });

  it('should provide accurate cost estimates', async () => {
    const command = createLLMRouterCommand();
    await executeCommand(command, ['cost', 'gpt-4o-mini', '--json', '--tokens', '1000000']);

    const jsonOutput = mockConsoleLog.mock.calls[0][0];
    const costEstimate = JSON.parse(jsonOutput);

    // GPT-4o-mini pricing: $0.15 input, $0.6 output per 1M tokens
    expect(costEstimate.inputCostPer1M).toBe(0.15);
    expect(costEstimate.outputCostPer1M).toBe(0.6);
    // At 1M tokens, cost should be: 0.15 + 0.6 = 0.75
    expect(costEstimate.estimatedTotalCost).toBeCloseTo(0.75, 2);
  });
});