mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Profa
271 lines
9.8 KiB
TypeScript
271 lines
9.8 KiB
TypeScript
/**
|
|
* Multi-Model Router Integration Tests
|
|
*
|
|
* Tests the agentic-flow Multi-Model Router capabilities:
|
|
* 1. Model optimization for different priorities (quality, cost, speed, privacy)
|
|
* 2. Cost tracking and budget constraints
|
|
* 3. Provider fallback and failover
|
|
* 4. MCP tool integration
|
|
* 5. CLI usage patterns
|
|
*/
|
|
|
|
import { describe, it, expect, beforeAll, afterAll } from '@jest/globals';
|
|
import { exec } from 'child_process';
|
|
import { promisify } from 'util';
|
|
|
|
const execAsync = promisify(exec);
|
|
|
|
describe('Multi-Model Router Integration', () => {
|
|
const TEST_TIMEOUT = 60000; // 60 seconds for API calls
|
|
|
|
beforeAll(() => {
|
|
// Verify environment variables are set
|
|
expect(process.env.OPENROUTER_API_KEY).toBeTruthy();
|
|
expect(process.env.ANTHROPIC_API_KEY).toBeTruthy();
|
|
});
|
|
|
|
describe('Model Optimization', () => {
|
|
it('should optimize for quality (flagship models)', async () => {
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --agent coder --task "Write secure authentication" --optimize --priority quality --verbose',
|
|
{ timeout: TEST_TIMEOUT }
|
|
);
|
|
|
|
expect(stdout).toContain('Optimized Model Selection');
|
|
expect(stdout).toMatch(/Claude Sonnet 4\.5|GPT-4o/);
|
|
expect(stdout).toContain('Quality:');
|
|
expect(stdout).toContain('flagship');
|
|
}, TEST_TIMEOUT);
|
|
|
|
it('should optimize for cost (cheapest models)', async () => {
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --agent researcher --task "Simple research" --optimize --priority cost --verbose',
|
|
{ timeout: TEST_TIMEOUT }
|
|
);
|
|
|
|
expect(stdout).toContain('Optimized Model Selection');
|
|
expect(stdout).toMatch(/DeepSeek|Llama|Gemini Flash/);
|
|
expect(stdout).toContain('Cost:');
|
|
}, TEST_TIMEOUT);
|
|
|
|
it('should optimize for speed (fastest models)', async () => {
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --agent analyst --task "Quick analysis" --optimize --priority speed --verbose',
|
|
{ timeout: TEST_TIMEOUT }
|
|
);
|
|
|
|
expect(stdout).toContain('Optimized Model Selection');
|
|
expect(stdout).toContain('Speed:');
|
|
}, TEST_TIMEOUT);
|
|
|
|
it('should respect max cost constraint', async () => {
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --agent coder --task "Simple function" --optimize --max-cost 0.001 --verbose',
|
|
{ timeout: TEST_TIMEOUT }
|
|
);
|
|
|
|
expect(stdout).toContain('Optimized Model Selection');
|
|
expect(stdout).toContain('Cost:');
|
|
|
|
// Extract cost estimate and verify it's below max
|
|
const costMatch = stdout.match(/Estimated cost: \$([0-9.]+)/);
|
|
if (costMatch) {
|
|
const estimatedCost = parseFloat(costMatch[1]);
|
|
expect(estimatedCost).toBeLessThanOrEqual(0.001);
|
|
}
|
|
}, TEST_TIMEOUT);
|
|
|
|
it('should optimize for privacy (local ONNX models)', async () => {
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --agent coder --task "Simple task" --optimize --priority privacy --verbose',
|
|
{ timeout: TEST_TIMEOUT }
|
|
);
|
|
|
|
expect(stdout).toContain('Optimized Model Selection');
|
|
expect(stdout).toMatch(/ONNX|local/);
|
|
expect(stdout).toContain('privacy');
|
|
}, TEST_TIMEOUT);
|
|
});
|
|
|
|
describe('MCP Tool Integration', () => {
|
|
it('should list agentic-flow MCP tools', async () => {
|
|
const { stdout } = await execAsync('agentic-flow mcp list');
|
|
|
|
expect(stdout).toContain('agentic-flow');
|
|
expect(stdout).toContain('agentic_flow_agent');
|
|
expect(stdout).toContain('agentic_flow_optimize_model');
|
|
});
|
|
|
|
it('should provide model recommendations via MCP', async () => {
|
|
// Note: This would typically be called via MCP client
|
|
// For now, we verify the CLI equivalent works
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --agent coder --task "Test task" --optimize --priority balanced --verbose'
|
|
);
|
|
|
|
expect(stdout).toContain('Optimized Model Selection');
|
|
expect(stdout).toContain('Reasoning:');
|
|
expect(stdout).toContain('Overall:');
|
|
});
|
|
});
|
|
|
|
describe('Provider Fallback', () => {
|
|
it('should handle provider fallback configuration', async () => {
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --help | grep -A 10 "PROVIDER FALLBACK"'
|
|
);
|
|
|
|
expect(stdout).toContain('Automatic failover');
|
|
expect(stdout).toContain('Circuit breaker');
|
|
expect(stdout).toContain('Cost optimization');
|
|
expect(stdout).toContain('Health monitoring');
|
|
});
|
|
|
|
it('should show fallback chain in verbose mode', async () => {
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --agent coder --task "Test" --optimize --verbose'
|
|
);
|
|
|
|
expect(stdout).toContain('Provider Selection Debug:');
|
|
expect(stdout).toMatch(/OpenRouter|Gemini|ONNX/);
|
|
});
|
|
});
|
|
|
|
describe('Cost Tracking', () => {
|
|
it('should display cost estimates in optimization output', async () => {
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --agent coder --task "Test function" --optimize --priority balanced --verbose'
|
|
);
|
|
|
|
expect(stdout).toContain('Cost:');
|
|
expect(stdout).toMatch(/\$[0-9.]+\/1M/);
|
|
expect(stdout).toContain('Estimated cost:');
|
|
});
|
|
|
|
it('should show cost savings comparison', async () => {
|
|
const qualityOutput = await execAsync(
|
|
'agentic-flow --agent coder --task "Test" --optimize --priority quality --verbose'
|
|
);
|
|
|
|
const costOutput = await execAsync(
|
|
'agentic-flow --agent coder --task "Test" --optimize --priority cost --verbose'
|
|
);
|
|
|
|
// Both should show cost information
|
|
expect(qualityOutput.stdout).toContain('Cost:');
|
|
expect(costOutput.stdout).toContain('Cost:');
|
|
|
|
// Cost-optimized should mention savings
|
|
expect(costOutput.stdout).toMatch(/cheaper|savings|85-98%/i);
|
|
});
|
|
});
|
|
|
|
describe('Agent-Specific Optimization', () => {
|
|
it('should optimize differently for coder agents (needs quality)', async () => {
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --agent coder --task "Complex algorithm" --optimize --priority balanced --verbose'
|
|
);
|
|
|
|
expect(stdout).toContain('coder agent');
|
|
expect(stdout).toMatch(/Claude|GPT-4/); // Should prefer quality models
|
|
});
|
|
|
|
it('should optimize differently for researcher agents (flexible)', async () => {
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --agent researcher --task "Research topic" --optimize --priority balanced --verbose'
|
|
);
|
|
|
|
expect(stdout).toContain('researcher');
|
|
// Researcher can use cheaper models
|
|
expect(stdout).toMatch(/DeepSeek|Llama|Gemini/i);
|
|
});
|
|
});
|
|
|
|
describe('Configuration and Environment', () => {
|
|
it('should respect environment variables', async () => {
|
|
const { stdout } = await execAsync('env | grep -E "ANTHROPIC_API_KEY|OPENROUTER_API_KEY"');
|
|
|
|
expect(stdout).toContain('ANTHROPIC_API_KEY=');
|
|
expect(stdout).toContain('OPENROUTER_API_KEY=');
|
|
});
|
|
|
|
it('should show available models', async () => {
|
|
const { stdout } = await execAsync('agentic-flow --help');
|
|
|
|
expect(stdout).toContain('OPENROUTER MODELS');
|
|
expect(stdout).toContain('deepseek');
|
|
expect(stdout).toContain('meta-llama');
|
|
expect(stdout).toContain('claude');
|
|
expect(stdout).toContain('gpt-4');
|
|
});
|
|
|
|
it('should support QUIC transport configuration', async () => {
|
|
const { stdout } = await execAsync('env | grep QUIC');
|
|
|
|
expect(stdout).toMatch(/QUIC_PORT|AGENTIC_FLOW_TRANSPORT/);
|
|
});
|
|
});
|
|
|
|
describe('Real-World Usage Patterns', () => {
|
|
it('should handle QE agent test generation with cost optimization', async () => {
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --agent qe-test-generator --task "Generate unit tests for auth" --optimize --priority cost --max-cost 0.002 --verbose',
|
|
{ timeout: TEST_TIMEOUT }
|
|
);
|
|
|
|
expect(stdout).toContain('Optimized Model Selection');
|
|
expect(stdout).toContain('qe-test-generator');
|
|
expect(stdout).toMatch(/test|Test/);
|
|
}, TEST_TIMEOUT);
|
|
|
|
it('should handle QE agent security scan with quality optimization', async () => {
|
|
const { stdout } = await execAsync(
|
|
'agentic-flow --agent qe-security-scanner --task "Security audit" --optimize --priority quality --verbose',
|
|
{ timeout: TEST_TIMEOUT }
|
|
);
|
|
|
|
expect(stdout).toContain('Optimized Model Selection');
|
|
expect(stdout).toContain('qe-security-scanner');
|
|
expect(stdout).toMatch(/security|Security/i);
|
|
}, TEST_TIMEOUT);
|
|
});
|
|
|
|
describe('Error Handling', () => {
|
|
it('should handle invalid priority gracefully', async () => {
|
|
try {
|
|
await execAsync(
|
|
'agentic-flow --agent coder --task "Test" --optimize --priority invalid'
|
|
);
|
|
fail('Should have thrown an error');
|
|
} catch (error: any) {
|
|
expect(error.message).toMatch(/priority|invalid/i);
|
|
}
|
|
});
|
|
|
|
it('should handle missing API keys gracefully', async () => {
|
|
// This test would require temporarily unsetting env vars
|
|
// Skipping for now as it would affect other tests
|
|
expect(true).toBe(true);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('Multi-Model Router CLI Examples', () => {
|
|
it('should demonstrate all CLI usage patterns', async () => {
|
|
const examples = [
|
|
'agentic-flow --agent coder --task "Build API" --optimize',
|
|
'agentic-flow --agent coder --task "Build API" --optimize --priority cost',
|
|
'agentic-flow --agent reviewer --task "Security audit" --optimize --priority quality',
|
|
'agentic-flow --agent coder --task "Simple function" --optimize --max-cost 0.001',
|
|
];
|
|
|
|
// Just verify the help text contains these patterns
|
|
const { stdout } = await execAsync('agentic-flow --help');
|
|
|
|
examples.forEach(example => {
|
|
const pattern = example.replace(/"/g, '');
|
|
expect(stdout).toContain('--optimize');
|
|
});
|
|
});
|
|
});
|