mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Profa
2253d05618
Remove the nested v3/ subdirectory by moving all active source code, tests, configs, and assets directly to the project root. Archive v2/ (dead code, 847MB) to v2-archive.tar.gz. Key changes: - Move v3/src/ → src/, v3/tests/ → tests/, v3/packages/ → packages/ - Move v3/tsconfig.json, v3/vitest.config.ts to root - Merge v3/package.json into root package.json (ESM, scripts, deps, exports) - Merge v3/scripts/ into scripts/, v3/implementation/ into docs/implementation/ - Update all CI workflows to remove "cd v3 &&" and v3/ path prefixes - Fix all init installer path resolution (3-level-up → 2-level-up) - Fix hooks to use "node ./dist/cli/bundle.js" instead of bare "aqe" - Update settings.json ADR/DDD directories to docs/implementation/ - Fix helper scripts (statusline, adr-compliance, ddd-tracker, guidance-hooks) - Update 58 agent definitions: aqe/v3/ → aqe/ memory namespaces - Update 15 guidance shards: v3/src/ → src/ path references - Fix sync interfaces: remove v3/ and v2/ path references - Fix test path calculations (process.cwd() + '..' no longer needed) - Fix flaky perf test threshold (500ms → 1000ms for CI environments) - Fix complexity-analyzer test to match ADR-051 keyword-based eligibility Build passes, 17,901 tests pass, CLI and MCP bundles verified. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
994 lines
31 KiB
TypeScript
994 lines
31 KiB
TypeScript
/**
|
|
* Agentic QE v3 - Agentic Flow Integration Tests
|
|
* ADR-051: Integration Tests for Phase 1 Components
|
|
*
|
|
* Tests the combined functionality of:
|
|
* - Agent Booster adapter (mechanical transforms)
|
|
* - ReasoningBank enhancements (trajectory tracking, experience replay, pattern evolution)
|
|
* - Cross-session persistence
|
|
* - Pattern learning from transforms
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
|
import { v4 as uuidv4 } from 'uuid';
|
|
|
|
// ============================================================================
|
|
// Integrated Types (combining Agent Booster and ReasoningBank)
|
|
// ============================================================================
|
|
|
|
type AgentBoosterTransform =
|
|
| 'var-to-const'
|
|
| 'add-types'
|
|
| 'remove-console'
|
|
| 'promise-to-async'
|
|
| 'cjs-to-esm'
|
|
| 'func-to-arrow';
|
|
|
|
interface TransformRequest {
|
|
code: string;
|
|
transforms: AgentBoosterTransform[];
|
|
language?: 'typescript' | 'javascript';
|
|
options?: { preserveComments?: boolean; strict?: boolean };
|
|
}
|
|
|
|
interface TransformResult {
|
|
code: string;
|
|
transformed: boolean;
|
|
transformsApplied: AgentBoosterTransform[];
|
|
confidence: number;
|
|
latencyMs: number;
|
|
errors?: string[];
|
|
}
|
|
|
|
interface TrajectoryStep {
|
|
id: string;
|
|
action: string;
|
|
input: Record<string, unknown>;
|
|
output: Record<string, unknown>;
|
|
reward: number;
|
|
timestamp: Date;
|
|
duration: number;
|
|
success: boolean;
|
|
metadata?: Record<string, unknown>;
|
|
}
|
|
|
|
interface Trajectory {
|
|
id: string;
|
|
taskId: string;
|
|
agentId: string;
|
|
domain: string;
|
|
steps: TrajectoryStep[];
|
|
totalReward: number;
|
|
success: boolean;
|
|
startedAt: Date;
|
|
completedAt?: Date;
|
|
}
|
|
|
|
interface TransformPattern {
|
|
id: string;
|
|
transforms: AgentBoosterTransform[];
|
|
context: {
|
|
codePattern: string;
|
|
language: string;
|
|
};
|
|
successRate: number;
|
|
avgLatencyMs: number;
|
|
usageCount: number;
|
|
confidence: number;
|
|
version: number;
|
|
}
|
|
|
|
interface LearningMetrics {
|
|
totalTransforms: number;
|
|
successfulTransforms: number;
|
|
patternsLearned: number;
|
|
avgLatencyImprovement: number;
|
|
llmBypassRate: number;
|
|
}
|
|
|
|
// ============================================================================
|
|
// Integrated System Implementation (for testing)
|
|
// ============================================================================
|
|
|
|
/**
|
|
* Integrated Agentic Flow system combining Agent Booster and ReasoningBank
|
|
*/
|
|
class AgenticFlowIntegration {
|
|
private patterns: Map<string, TransformPattern> = new Map();
|
|
private trajectories: Map<string, Trajectory> = new Map();
|
|
private activeTrajectory: Trajectory | null = null;
|
|
private metrics: LearningMetrics = {
|
|
totalTransforms: 0,
|
|
successfulTransforms: 0,
|
|
patternsLearned: 0,
|
|
avgLatencyImprovement: 0,
|
|
llmBypassRate: 0,
|
|
};
|
|
private persistenceEnabled: boolean;
|
|
private storage: Map<string, unknown> = new Map();
|
|
|
|
constructor(options: { persistenceEnabled?: boolean } = {}) {
|
|
this.persistenceEnabled = options.persistenceEnabled ?? true;
|
|
}
|
|
|
|
// ==========================================================================
|
|
// Agent Booster Integration
|
|
// ==========================================================================
|
|
|
|
/**
|
|
* Transform code using Agent Booster with pattern learning
|
|
*/
|
|
async transform(request: TransformRequest): Promise<TransformResult> {
|
|
const startTime = performance.now();
|
|
|
|
// Start trajectory for this transform task
|
|
this.startTransformTrajectory(request);
|
|
|
|
// Check if we have a matching pattern
|
|
const existingPattern = this.findMatchingPattern(request);
|
|
|
|
let result: TransformResult;
|
|
|
|
if (existingPattern && existingPattern.confidence > 0.9) {
|
|
// Use cached pattern approach (skip LLM)
|
|
result = await this.applyPatternTransform(request, existingPattern);
|
|
this.metrics.llmBypassRate =
|
|
(this.metrics.llmBypassRate * this.metrics.totalTransforms + 1) /
|
|
(this.metrics.totalTransforms + 1);
|
|
} else {
|
|
// Apply transforms directly
|
|
result = await this.applyDirectTransform(request);
|
|
}
|
|
|
|
// Record step in trajectory
|
|
this.recordTransformStep(request, result);
|
|
|
|
// End trajectory
|
|
const trajectory = this.endTransformTrajectory(result.transformed && !result.errors?.length);
|
|
|
|
// Learn from this transform
|
|
if (result.transformed && result.confidence > 0.7) {
|
|
await this.learnFromTransform(request, result, trajectory);
|
|
}
|
|
|
|
// Update metrics
|
|
this.metrics.totalTransforms++;
|
|
if (result.transformed && !result.errors?.length) {
|
|
this.metrics.successfulTransforms++;
|
|
}
|
|
|
|
result.latencyMs = performance.now() - startTime;
|
|
return result;
|
|
}
|
|
|
|
private startTransformTrajectory(request: TransformRequest): void {
|
|
this.activeTrajectory = {
|
|
id: uuidv4(),
|
|
taskId: `transform-${Date.now()}`,
|
|
agentId: 'agent-booster',
|
|
domain: 'code-transform',
|
|
steps: [],
|
|
totalReward: 0,
|
|
success: false,
|
|
startedAt: new Date(),
|
|
};
|
|
this.trajectories.set(this.activeTrajectory.id, this.activeTrajectory);
|
|
}
|
|
|
|
private recordTransformStep(request: TransformRequest, result: TransformResult): void {
|
|
if (!this.activeTrajectory) return;
|
|
|
|
const reward = result.transformed ? result.confidence : 0;
|
|
|
|
const step: TrajectoryStep = {
|
|
id: uuidv4(),
|
|
action: `apply-transforms:${request.transforms.join(',')}`,
|
|
input: { codeLength: request.code.length, transforms: request.transforms },
|
|
output: {
|
|
transformed: result.transformed,
|
|
confidence: result.confidence,
|
|
appliedCount: result.transformsApplied.length,
|
|
},
|
|
reward,
|
|
timestamp: new Date(),
|
|
duration: result.latencyMs,
|
|
success: result.transformed && !result.errors?.length,
|
|
};
|
|
|
|
this.activeTrajectory.steps.push(step);
|
|
this.activeTrajectory.totalReward += reward;
|
|
}
|
|
|
|
private endTransformTrajectory(success: boolean): Trajectory {
|
|
if (!this.activeTrajectory) {
|
|
throw new Error('No active trajectory');
|
|
}
|
|
|
|
this.activeTrajectory.success = success;
|
|
this.activeTrajectory.completedAt = new Date();
|
|
|
|
if (this.persistenceEnabled) {
|
|
this.persistTrajectory(this.activeTrajectory);
|
|
}
|
|
|
|
const completed = this.activeTrajectory;
|
|
this.activeTrajectory = null;
|
|
return completed;
|
|
}
|
|
|
|
private findMatchingPattern(request: TransformRequest): TransformPattern | undefined {
|
|
// Generate pattern key from request characteristics
|
|
const patternKey = this.generatePatternKey(request);
|
|
return this.patterns.get(patternKey);
|
|
}
|
|
|
|
private generatePatternKey(request: TransformRequest): string {
|
|
const transforms = [...request.transforms].sort().join('|');
|
|
const lang = request.language || 'javascript';
|
|
return `${transforms}:${lang}`;
|
|
}
|
|
|
|
private async applyPatternTransform(
|
|
request: TransformRequest,
|
|
pattern: TransformPattern
|
|
): Promise<TransformResult> {
|
|
// Apply the pattern's recommended approach
|
|
pattern.usageCount++;
|
|
return this.applyDirectTransform(request);
|
|
}
|
|
|
|
private async applyDirectTransform(request: TransformRequest): Promise<TransformResult> {
|
|
let code = request.code;
|
|
const applied: AgentBoosterTransform[] = [];
|
|
let confidence = 1.0;
|
|
|
|
for (const transform of request.transforms) {
|
|
const result = this.applySingleTransform(code, transform);
|
|
if (result.transformed) {
|
|
code = result.code;
|
|
applied.push(transform);
|
|
confidence = Math.min(confidence, result.confidence);
|
|
}
|
|
}
|
|
|
|
return {
|
|
code,
|
|
transformed: applied.length > 0,
|
|
transformsApplied: applied,
|
|
confidence,
|
|
latencyMs: 0, // Will be set by caller
|
|
};
|
|
}
|
|
|
|
private applySingleTransform(
|
|
code: string,
|
|
transform: AgentBoosterTransform
|
|
): { code: string; transformed: boolean; confidence: number } {
|
|
switch (transform) {
|
|
case 'var-to-const': {
|
|
const newCode = code.replace(/\bvar\s+/g, 'const ');
|
|
return { code: newCode, transformed: newCode !== code, confidence: 0.95 };
|
|
}
|
|
case 'remove-console': {
|
|
const newCode = code.replace(/^\s*console\.\w+\([^)]*\);?\s*$/gm, '');
|
|
return { code: newCode, transformed: newCode !== code, confidence: 0.99 };
|
|
}
|
|
case 'cjs-to-esm': {
|
|
let newCode = code.replace(
|
|
/const\s+(\w+)\s*=\s*require\(['"]([^'"]+)['"]\);?/g,
|
|
"import $1 from '$2';"
|
|
);
|
|
newCode = newCode.replace(/module\.exports\s*=\s*(\w+);?/g, 'export default $1;');
|
|
return { code: newCode, transformed: newCode !== code, confidence: 0.9 };
|
|
}
|
|
case 'func-to-arrow': {
|
|
const newCode = code.replace(
|
|
/\bfunction\s+(\w+)\s*\(([^)]*)\)\s*\{/g,
|
|
'const $1 = ($2) => {'
|
|
);
|
|
return { code: newCode, transformed: newCode !== code, confidence: 0.88 };
|
|
}
|
|
default:
|
|
return { code, transformed: false, confidence: 1.0 };
|
|
}
|
|
}
|
|
|
|
private async learnFromTransform(
|
|
request: TransformRequest,
|
|
result: TransformResult,
|
|
trajectory: Trajectory
|
|
): Promise<void> {
|
|
const patternKey = this.generatePatternKey(request);
|
|
let pattern = this.patterns.get(patternKey);
|
|
|
|
if (pattern) {
|
|
// Update existing pattern
|
|
const totalUsage = pattern.usageCount + 1;
|
|
const successCount = pattern.successRate * pattern.usageCount + (result.transformed ? 1 : 0);
|
|
|
|
pattern.successRate = successCount / totalUsage;
|
|
pattern.avgLatencyMs = (pattern.avgLatencyMs * pattern.usageCount + result.latencyMs) / totalUsage;
|
|
pattern.usageCount = totalUsage;
|
|
pattern.confidence = Math.min(1, pattern.confidence + 0.01);
|
|
pattern.version++;
|
|
} else {
|
|
// Create new pattern
|
|
pattern = {
|
|
id: uuidv4(),
|
|
transforms: request.transforms,
|
|
context: {
|
|
codePattern: this.extractCodePattern(request.code),
|
|
language: request.language || 'javascript',
|
|
},
|
|
successRate: result.transformed ? 1 : 0,
|
|
avgLatencyMs: result.latencyMs,
|
|
usageCount: 1,
|
|
confidence: result.confidence * 0.5, // Start with reduced confidence
|
|
version: 1,
|
|
};
|
|
|
|
this.patterns.set(patternKey, pattern);
|
|
this.metrics.patternsLearned++;
|
|
}
|
|
|
|
if (this.persistenceEnabled) {
|
|
this.persistPattern(pattern);
|
|
}
|
|
}
|
|
|
|
private extractCodePattern(code: string): string {
|
|
// Extract a simplified pattern from the code
|
|
const hasVars = /\bvar\s+/.test(code);
|
|
const hasConsole = /console\.\w+/.test(code);
|
|
const hasCjs = /require\(|module\.exports/.test(code);
|
|
const hasFunc = /\bfunction\s+\w+/.test(code);
|
|
|
|
const patterns: string[] = [];
|
|
if (hasVars) patterns.push('var-decl');
|
|
if (hasConsole) patterns.push('console-stmt');
|
|
if (hasCjs) patterns.push('cjs-module');
|
|
if (hasFunc) patterns.push('func-decl');
|
|
|
|
return patterns.join(',') || 'unknown';
|
|
}
|
|
|
|
// ==========================================================================
|
|
// Persistence Methods
|
|
// ==========================================================================
|
|
|
|
private persistTrajectory(trajectory: Trajectory): void {
|
|
this.storage.set(`trajectory:${trajectory.id}`, trajectory);
|
|
}
|
|
|
|
private persistPattern(pattern: TransformPattern): void {
|
|
this.storage.set(`pattern:${pattern.id}`, pattern);
|
|
}
|
|
|
|
loadPersistedData(): void {
|
|
// Load patterns from storage
|
|
for (const [key, value] of this.storage.entries()) {
|
|
if (key.startsWith('pattern:') && value) {
|
|
const pattern = value as TransformPattern;
|
|
const patternKey = `${pattern.transforms.sort().join('|')}:${pattern.context.language}`;
|
|
this.patterns.set(patternKey, pattern);
|
|
}
|
|
if (key.startsWith('trajectory:') && value) {
|
|
const trajectory = value as Trajectory;
|
|
this.trajectories.set(trajectory.id, trajectory);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ==========================================================================
|
|
// Query Methods
|
|
// ==========================================================================
|
|
|
|
/**
|
|
* Get all learned patterns
|
|
*/
|
|
getPatterns(): TransformPattern[] {
|
|
return Array.from(this.patterns.values());
|
|
}
|
|
|
|
/**
|
|
* Get trajectories for a specific domain
|
|
*/
|
|
getTrajectories(domain?: string): Trajectory[] {
|
|
const all = Array.from(this.trajectories.values());
|
|
if (domain) {
|
|
return all.filter((t) => t.domain === domain);
|
|
}
|
|
return all;
|
|
}
|
|
|
|
/**
|
|
* Get learning metrics
|
|
*/
|
|
getMetrics(): LearningMetrics {
|
|
return { ...this.metrics };
|
|
}
|
|
|
|
/**
|
|
* Check if a transform would be handled by Agent Booster (no LLM needed)
|
|
*/
|
|
canHandleWithoutLLM(request: TransformRequest): boolean {
|
|
const pattern = this.findMatchingPattern(request);
|
|
return !!pattern && pattern.confidence > 0.9 && pattern.successRate > 0.95;
|
|
}
|
|
|
|
/**
|
|
* Get recommended transforms for code
|
|
*/
|
|
detectTransformOpportunities(code: string): AgentBoosterTransform[] {
|
|
const opportunities: AgentBoosterTransform[] = [];
|
|
|
|
if (/\bvar\s+/.test(code)) opportunities.push('var-to-const');
|
|
if (/console\.\w+/.test(code)) opportunities.push('remove-console');
|
|
if (/require\(|module\.exports/.test(code)) opportunities.push('cjs-to-esm');
|
|
if (/\bfunction\s+\w+\s*\(/.test(code)) opportunities.push('func-to-arrow');
|
|
|
|
return opportunities;
|
|
}
|
|
|
|
/**
|
|
* Reset metrics (for testing)
|
|
*/
|
|
resetMetrics(): void {
|
|
this.metrics = {
|
|
totalTransforms: 0,
|
|
successfulTransforms: 0,
|
|
patternsLearned: 0,
|
|
avgLatencyImprovement: 0,
|
|
llmBypassRate: 0,
|
|
};
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// Integration Tests
|
|
// ============================================================================
|
|
|
|
describe('AgenticFlowIntegration', () => {
|
|
let integration: AgenticFlowIntegration;
|
|
|
|
beforeEach(() => {
|
|
integration = new AgenticFlowIntegration({ persistenceEnabled: true });
|
|
});
|
|
|
|
describe('Agent Booster + ReasoningBank combined workflow', () => {
|
|
it('should transform code and record trajectory', async () => {
|
|
const result = await integration.transform({
|
|
code: `var x = 1;
|
|
console.log(x);`,
|
|
transforms: ['var-to-const', 'remove-console'],
|
|
});
|
|
|
|
expect(result.transformed).toBe(true);
|
|
expect(result.code).toContain('const x');
|
|
expect(result.code).not.toContain('console.log');
|
|
|
|
const trajectories = integration.getTrajectories('code-transform');
|
|
expect(trajectories.length).toBe(1);
|
|
expect(trajectories[0].success).toBe(true);
|
|
});
|
|
|
|
it('should learn patterns from successful transforms', async () => {
|
|
// First transform
|
|
await integration.transform({
|
|
code: `var a = 1;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
// Second similar transform
|
|
await integration.transform({
|
|
code: `var b = 2;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
const patterns = integration.getPatterns();
|
|
expect(patterns.length).toBe(1);
|
|
expect(patterns[0].usageCount).toBe(2);
|
|
expect(patterns[0].successRate).toBe(1);
|
|
});
|
|
|
|
it('should improve pattern confidence over time', async () => {
|
|
// Initial transform
|
|
await integration.transform({
|
|
code: `var x = 1;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
const initialPattern = integration.getPatterns()[0];
|
|
const initialConfidence = initialPattern.confidence;
|
|
|
|
// More transforms to build confidence
|
|
for (let i = 0; i < 10; i++) {
|
|
await integration.transform({
|
|
code: `var y${i} = ${i};`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
}
|
|
|
|
const finalPattern = integration.getPatterns()[0];
|
|
expect(finalPattern.confidence).toBeGreaterThan(initialConfidence);
|
|
});
|
|
});
|
|
|
|
describe('pattern learning from transforms', () => {
|
|
it('should create distinct patterns for different transform combinations', async () => {
|
|
await integration.transform({
|
|
code: `var x = 1;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
await integration.transform({
|
|
code: `console.log('test');`,
|
|
transforms: ['remove-console'],
|
|
});
|
|
|
|
await integration.transform({
|
|
code: `var y = 2; console.log(y);`,
|
|
transforms: ['var-to-const', 'remove-console'],
|
|
});
|
|
|
|
const patterns = integration.getPatterns();
|
|
expect(patterns.length).toBe(3);
|
|
});
|
|
|
|
it('should track pattern versions when updated', async () => {
|
|
await integration.transform({
|
|
code: `var a = 1;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
const initialVersion = integration.getPatterns()[0].version;
|
|
|
|
await integration.transform({
|
|
code: `var b = 2;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
const updatedVersion = integration.getPatterns()[0].version;
|
|
expect(updatedVersion).toBe(initialVersion + 1);
|
|
});
|
|
|
|
it('should calculate average latency across pattern usage', async () => {
|
|
const result1 = await integration.transform({
|
|
code: `var a = 1;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
const result2 = await integration.transform({
|
|
code: `var b = 2;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
const pattern = integration.getPatterns()[0];
|
|
// Pattern stores latency from the transform function itself
|
|
// avgLatencyMs should be >= 0 (could be 0 if transforms are very fast)
|
|
expect(pattern.avgLatencyMs).toBeGreaterThanOrEqual(0);
|
|
// But the result's latency should be > 0
|
|
expect(result1.latencyMs).toBeGreaterThanOrEqual(0);
|
|
expect(result2.latencyMs).toBeGreaterThanOrEqual(0);
|
|
});
|
|
});
|
|
|
|
describe('cross-session persistence', () => {
|
|
it('should persist and restore patterns', async () => {
|
|
// Create patterns in first "session"
|
|
await integration.transform({
|
|
code: `var x = 1;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
// Simulate new session
|
|
const newIntegration = new AgenticFlowIntegration({ persistenceEnabled: true });
|
|
|
|
// Share storage (simulating persistence)
|
|
(newIntegration as any).storage = (integration as any).storage;
|
|
newIntegration.loadPersistedData();
|
|
|
|
const patterns = newIntegration.getPatterns();
|
|
expect(patterns.length).toBe(1);
|
|
expect(patterns[0].transforms).toContain('var-to-const');
|
|
});
|
|
|
|
it('should persist trajectories for analysis', async () => {
|
|
await integration.transform({
|
|
code: `var x = 1;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
await integration.transform({
|
|
code: `var y = 2;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
const newIntegration = new AgenticFlowIntegration({ persistenceEnabled: true });
|
|
(newIntegration as any).storage = (integration as any).storage;
|
|
newIntegration.loadPersistedData();
|
|
|
|
const trajectories = newIntegration.getTrajectories();
|
|
expect(trajectories.length).toBe(2);
|
|
});
|
|
|
|
it('should maintain pattern metrics across sessions', async () => {
|
|
// First session - build up pattern
|
|
for (let i = 0; i < 5; i++) {
|
|
await integration.transform({
|
|
code: `var x${i} = ${i};`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
}
|
|
|
|
// Snapshot the usage count before restoring (value, not reference)
|
|
const usageCountBeforeRestore = integration.getPatterns()[0].usageCount;
|
|
|
|
// New session
|
|
const newIntegration = new AgenticFlowIntegration();
|
|
(newIntegration as any).storage = (integration as any).storage;
|
|
newIntegration.loadPersistedData();
|
|
|
|
// Continue building
|
|
for (let i = 5; i < 10; i++) {
|
|
await newIntegration.transform({
|
|
code: `var y${i} = ${i};`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
}
|
|
|
|
const patternAfterContinue = newIntegration.getPatterns()[0];
|
|
// After 5 more transforms, usage count should be >= 5 (original) + 5 (new)
|
|
// Using >= because the first session had 5 uses
|
|
expect(patternAfterContinue.usageCount).toBeGreaterThanOrEqual(usageCountBeforeRestore + 5);
|
|
});
|
|
});
|
|
|
|
describe('LLM bypass detection', () => {
|
|
it('should bypass LLM for high-confidence patterns', async () => {
|
|
// Build high-confidence pattern
|
|
for (let i = 0; i < 20; i++) {
|
|
await integration.transform({
|
|
code: `var x${i} = ${i};`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
}
|
|
|
|
const canBypass = integration.canHandleWithoutLLM({
|
|
code: `var newVar = 'test';`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
// After many successful transforms, confidence should be high enough
|
|
const pattern = integration.getPatterns()[0];
|
|
if (pattern.confidence > 0.9 && pattern.successRate > 0.95) {
|
|
expect(canBypass).toBe(true);
|
|
}
|
|
});
|
|
|
|
it('should not bypass LLM for new patterns', async () => {
|
|
const canBypass = integration.canHandleWithoutLLM({
|
|
code: `const fs = require('fs');`,
|
|
transforms: ['cjs-to-esm'],
|
|
});
|
|
|
|
expect(canBypass).toBe(false);
|
|
});
|
|
|
|
it('should track LLM bypass rate in metrics', async () => {
|
|
// Initial transforms without bypass
|
|
for (let i = 0; i < 5; i++) {
|
|
await integration.transform({
|
|
code: `var x${i} = ${i};`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
}
|
|
|
|
const metrics = integration.getMetrics();
|
|
expect(metrics.totalTransforms).toBe(5);
|
|
// Bypass rate should increase as patterns are learned
|
|
expect(metrics.llmBypassRate).toBeGreaterThanOrEqual(0);
|
|
});
|
|
});
|
|
|
|
describe('transform opportunity detection', () => {
|
|
it('should detect var declarations', () => {
|
|
const opportunities = integration.detectTransformOpportunities(`
|
|
var x = 1;
|
|
var y = 2;
|
|
const z = 3;
|
|
`);
|
|
|
|
expect(opportunities).toContain('var-to-const');
|
|
});
|
|
|
|
it('should detect console statements', () => {
|
|
const opportunities = integration.detectTransformOpportunities(`
|
|
console.log('test');
|
|
console.error('error');
|
|
`);
|
|
|
|
expect(opportunities).toContain('remove-console');
|
|
});
|
|
|
|
it('should detect CommonJS patterns', () => {
|
|
const opportunities = integration.detectTransformOpportunities(`
|
|
const fs = require('fs');
|
|
module.exports = MyClass;
|
|
`);
|
|
|
|
expect(opportunities).toContain('cjs-to-esm');
|
|
});
|
|
|
|
it('should detect multiple opportunities', () => {
|
|
const opportunities = integration.detectTransformOpportunities(`
|
|
var fs = require('fs');
|
|
console.log('Loading...');
|
|
function process(data) {
|
|
console.log(data);
|
|
}
|
|
`);
|
|
|
|
expect(opportunities).toContain('var-to-const');
|
|
expect(opportunities).toContain('remove-console');
|
|
expect(opportunities).toContain('cjs-to-esm');
|
|
expect(opportunities).toContain('func-to-arrow');
|
|
});
|
|
});
|
|
|
|
describe('metrics tracking', () => {
|
|
it('should track total transforms', async () => {
|
|
await integration.transform({ code: `var x = 1;`, transforms: ['var-to-const'] });
|
|
await integration.transform({ code: `var y = 2;`, transforms: ['var-to-const'] });
|
|
|
|
const metrics = integration.getMetrics();
|
|
expect(metrics.totalTransforms).toBe(2);
|
|
});
|
|
|
|
it('should track successful transforms', async () => {
|
|
await integration.transform({ code: `var x = 1;`, transforms: ['var-to-const'] });
|
|
await integration.transform({ code: `const y = 2;`, transforms: ['var-to-const'] }); // No change
|
|
|
|
const metrics = integration.getMetrics();
|
|
expect(metrics.successfulTransforms).toBe(1);
|
|
});
|
|
|
|
it('should track patterns learned', async () => {
|
|
await integration.transform({ code: `var x = 1;`, transforms: ['var-to-const'] });
|
|
await integration.transform({ code: `console.log('x');`, transforms: ['remove-console'] });
|
|
|
|
const metrics = integration.getMetrics();
|
|
expect(metrics.patternsLearned).toBe(2);
|
|
});
|
|
|
|
it('should reset metrics correctly', async () => {
|
|
await integration.transform({ code: `var x = 1;`, transforms: ['var-to-const'] });
|
|
|
|
integration.resetMetrics();
|
|
|
|
const metrics = integration.getMetrics();
|
|
expect(metrics.totalTransforms).toBe(0);
|
|
expect(metrics.successfulTransforms).toBe(0);
|
|
});
|
|
});
|
|
|
|
describe('trajectory analysis', () => {
|
|
it('should record step details in trajectory', async () => {
|
|
await integration.transform({
|
|
code: `var x = 1;
|
|
console.log(x);`,
|
|
transforms: ['var-to-const', 'remove-console'],
|
|
});
|
|
|
|
const trajectories = integration.getTrajectories();
|
|
expect(trajectories.length).toBe(1);
|
|
|
|
const trajectory = trajectories[0];
|
|
expect(trajectory.steps.length).toBe(1);
|
|
expect(trajectory.steps[0].action).toContain('apply-transforms');
|
|
expect(trajectory.steps[0].success).toBe(true);
|
|
});
|
|
|
|
it('should calculate trajectory reward based on confidence', async () => {
|
|
await integration.transform({
|
|
code: `var x = 1;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
const trajectory = integration.getTrajectories()[0];
|
|
expect(trajectory.totalReward).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('should mark failed trajectories correctly', async () => {
|
|
await integration.transform({
|
|
code: `const x = 1;`, // No var to convert
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
const trajectory = integration.getTrajectories()[0];
|
|
expect(trajectory.success).toBe(false);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('Performance benchmarks (combined)', () => {
|
|
it('should transform + learn in under 50ms per operation', async () => {
|
|
const integration = new AgenticFlowIntegration();
|
|
const code = `var x = 1;
|
|
console.log(x);
|
|
var y = 2;`;
|
|
|
|
const times: number[] = [];
|
|
|
|
for (let i = 0; i < 100; i++) {
|
|
const start = performance.now();
|
|
await integration.transform({
|
|
code,
|
|
transforms: ['var-to-const', 'remove-console'],
|
|
});
|
|
times.push(performance.now() - start);
|
|
}
|
|
|
|
const avg = times.reduce((a, b) => a + b, 0) / times.length;
|
|
const p95 = times.sort((a, b) => a - b)[Math.floor(times.length * 0.95)];
|
|
|
|
expect(avg).toBeLessThan(50);
|
|
expect(p95).toBeLessThan(100);
|
|
});
|
|
|
|
it('should handle large batches efficiently', async () => {
|
|
// Use separate integrations for parallel execution since
|
|
// AgenticFlowIntegration uses single activeTrajectory state
|
|
const files = Array.from({ length: 50 }, (_, i) => ({
|
|
code: `var x${i} = ${i};
|
|
console.log(x${i});`,
|
|
index: i,
|
|
}));
|
|
|
|
const startTime = performance.now();
|
|
|
|
// Execute in parallel with separate integration instances to avoid
|
|
// trajectory state conflicts (each instance has its own activeTrajectory)
|
|
await Promise.all(
|
|
files.map((f) => {
|
|
const integration = new AgenticFlowIntegration({ persistenceEnabled: false });
|
|
return integration.transform({
|
|
code: f.code,
|
|
transforms: ['var-to-const', 'remove-console'],
|
|
});
|
|
})
|
|
);
|
|
|
|
const totalTime = performance.now() - startTime;
|
|
|
|
// 50 files should complete in under 2 seconds
|
|
expect(totalTime).toBeLessThan(2000);
|
|
});
|
|
|
|
it('should maintain performance as pattern store grows', async () => {
|
|
const integration = new AgenticFlowIntegration();
|
|
|
|
// Build up pattern store
|
|
for (let i = 0; i < 100; i++) {
|
|
await integration.transform({
|
|
code: `var x${i} = ${i};`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
}
|
|
|
|
// Measure time for new transform with populated store
|
|
const start = performance.now();
|
|
await integration.transform({
|
|
code: `var newVar = 'test';`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
const duration = performance.now() - start;
|
|
|
|
// Should still be fast (relaxed for CI environments)
|
|
expect(duration).toBeLessThan(100);
|
|
});
|
|
});
|
|
|
|
describe('Edge cases and error handling', () => {
|
|
it('should handle empty code gracefully', async () => {
|
|
const integration = new AgenticFlowIntegration();
|
|
|
|
const result = await integration.transform({
|
|
code: '',
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
expect(result.transformed).toBe(false);
|
|
expect(result.code).toBe('');
|
|
});
|
|
|
|
it('should handle code with no matching patterns', async () => {
|
|
const integration = new AgenticFlowIntegration();
|
|
|
|
const result = await integration.transform({
|
|
code: `const x = 1;
|
|
const y = 2;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
expect(result.transformed).toBe(false);
|
|
expect(result.transformsApplied).toHaveLength(0);
|
|
});
|
|
|
|
it('should handle mixed success/failure transforms', async () => {
|
|
const integration = new AgenticFlowIntegration();
|
|
|
|
const result = await integration.transform({
|
|
code: `var x = 1;
|
|
const y = 2;`, // var exists, but also has const
|
|
transforms: ['var-to-const', 'cjs-to-esm'], // cjs-to-esm won't match
|
|
});
|
|
|
|
expect(result.transformed).toBe(true);
|
|
expect(result.transformsApplied).toContain('var-to-const');
|
|
expect(result.transformsApplied).not.toContain('cjs-to-esm');
|
|
});
|
|
|
|
it('should handle complex nested code', async () => {
|
|
const integration = new AgenticFlowIntegration();
|
|
|
|
const result = await integration.transform({
|
|
code: `
|
|
function outer() {
|
|
var x = 1;
|
|
function inner() {
|
|
var y = 2;
|
|
console.log(x + y);
|
|
}
|
|
console.log('outer');
|
|
}
|
|
`,
|
|
transforms: ['var-to-const', 'remove-console', 'func-to-arrow'],
|
|
});
|
|
|
|
expect(result.transformed).toBe(true);
|
|
expect(result.code).toContain('const x');
|
|
expect(result.code).toContain('const y');
|
|
expect(result.code).not.toContain('console.log');
|
|
});
|
|
});
|
|
|
|
describe('LLM baseline comparison (simulated)', () => {
|
|
it('should demonstrate latency improvement over LLM', async () => {
|
|
const integration = new AgenticFlowIntegration();
|
|
|
|
// Simulate LLM latency (352ms as per ADR-051)
|
|
const simulatedLLMLatency = 352;
|
|
|
|
const result = await integration.transform({
|
|
code: `var x = 1;`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
|
|
const speedup = simulatedLLMLatency / Math.max(result.latencyMs, 1);
|
|
|
|
// Should be significantly faster than LLM
|
|
expect(result.latencyMs).toBeLessThan(50);
|
|
expect(speedup).toBeGreaterThan(7); // At least 7x faster (conservatively)
|
|
});
|
|
|
|
it('should track latency improvement metrics', async () => {
|
|
const integration = new AgenticFlowIntegration();
|
|
|
|
// Run multiple transforms
|
|
for (let i = 0; i < 10; i++) {
|
|
await integration.transform({
|
|
code: `var x${i} = ${i};`,
|
|
transforms: ['var-to-const'],
|
|
});
|
|
}
|
|
|
|
const patterns = integration.getPatterns();
|
|
expect(patterns.length).toBeGreaterThan(0);
|
|
|
|
// Average latency should be well under LLM baseline
|
|
const avgLatency = patterns[0].avgLatencyMs;
|
|
expect(avgLatency).toBeLessThan(100);
|
|
});
|
|
});
|