mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
b6fd679254
Found by the qe-devils-advocate session review, AFTER #569 was already committed. The handler was producing honest provenance; `coverageAnalyzeConfig.mapToResult` was converting it back into fabricated certainty on three paths. 1. Absent provenance was treated as "collected". `data.branchDataCollected !== false` — and `undefined !== false` is true. Any producer that supplies no provenance (notably the coverage-analysis domain plugin, whose handler returns a nested-`summary` CoverageReport) therefore surfaced issue #569's EXACT reported symptom: 100% branch coverage alongside 0% function coverage, labelled `measured: true`, with `confidence: 0.7` on its gaps. Now requires an explicit `true`; absence means unknown, so an unprovenanced result reports `null`. 2. `measured` was derived as `!estimated`. These are not complements — there is a third state where nothing was collected at all. An empty project reported `lineCoverage: 0, measured: true`: "we measured 0%" when the truth was "we measured nothing". Verified through the real MCP server before and after. `measured` now requires an explicit claim. 3. `null` was coerced to `0`, and risk was graded off it. `?? 0` turned "no data" into a plottable 0%, and `null < 70` is `false` in JS, so an unguarded comparison graded "no data" as 'low' risk — the most dangerous possible default. Nulls now survive; risk grading requires a real number. Also fixes the stale-result mystery that dogged verification all session. `src/optimization/session-cache.ts` fingerprints on domain+action+input only, and persists to kv_store with a 1-hour TTL, so entries survive package upgrades: a project analyzed before a fix keeps replaying the old answer, making the fix look broken. I twice mis-diagnosed this (first as a cache, then wrongly retracted that; it WAS the cache). The fingerprint now includes the build identity, so an upgrade misses cleanly and old entries age out. `AQE_SESSION_CACHE_SALT` forces a cold cache without deleting anything. This matters beyond #569: every fix in this release would have appeared not to work for any user with a warm cache. `CoverageAnalyzeResult.lineCoverage`/`statementCoverage` are now `number | null`, joining `branchCoverage`/`functionCoverage`. `0` means a measured zero; `null` means unknown. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
372 lines
12 KiB
TypeScript
372 lines
12 KiB
TypeScript
/**
|
|
* Unit Tests: SessionOperationCache
|
|
* Imp-15: Session Reuse for Repeated Operations
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
|
import {
|
|
SessionOperationCache,
|
|
getSessionCache,
|
|
resetSessionCache,
|
|
DEFAULT_SESSION_CACHE_CONFIG,
|
|
} from '../../../src/optimization/session-cache.js';
|
|
|
|
describe('SessionOperationCache', () => {
|
|
let cache: SessionOperationCache;
|
|
|
|
beforeEach(() => {
|
|
resetSessionCache();
|
|
cache = new SessionOperationCache({ persistToDb: false });
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
// ==========================================================================
|
|
// computeFingerprint
|
|
// ==========================================================================
|
|
|
|
describe('computeFingerprint', () => {
|
|
it('should produce consistent hashes for the same input', () => {
|
|
const fp1 = cache.computeFingerprint('test-gen', 'generate', { file: 'a.ts' });
|
|
const fp2 = cache.computeFingerprint('test-gen', 'generate', { file: 'a.ts' });
|
|
expect(fp1).toBe(fp2);
|
|
});
|
|
|
|
it('should produce different hashes for different inputs', () => {
|
|
const fp1 = cache.computeFingerprint('test-gen', 'generate', { file: 'a.ts' });
|
|
const fp2 = cache.computeFingerprint('test-gen', 'generate', { file: 'b.ts' });
|
|
expect(fp1).not.toBe(fp2);
|
|
});
|
|
|
|
it('should produce different hashes for different domains', () => {
|
|
const fp1 = cache.computeFingerprint('domain-a', 'action', {});
|
|
const fp2 = cache.computeFingerprint('domain-b', 'action', {});
|
|
expect(fp1).not.toBe(fp2);
|
|
});
|
|
|
|
it('should produce different hashes for different actions', () => {
|
|
const fp1 = cache.computeFingerprint('domain', 'action-a', {});
|
|
const fp2 = cache.computeFingerprint('domain', 'action-b', {});
|
|
expect(fp1).not.toBe(fp2);
|
|
});
|
|
|
|
it('should return a 16-character hex string', () => {
|
|
const fp = cache.computeFingerprint('d', 'a', { x: 1 });
|
|
expect(fp).toMatch(/^[0-9a-f]{16}$/);
|
|
});
|
|
});
|
|
|
|
// ==========================================================================
|
|
// get (cache miss)
|
|
// ==========================================================================
|
|
|
|
describe('get (miss)', () => {
|
|
it('should return null on miss', () => {
|
|
const result = cache.get('nonexistent');
|
|
expect(result).toBeNull();
|
|
});
|
|
|
|
it('should increment misses counter on miss', () => {
|
|
cache.get('nonexistent');
|
|
cache.get('also-nonexistent');
|
|
const stats = cache.getStats();
|
|
expect(stats.misses).toBe(2);
|
|
expect(stats.hits).toBe(0);
|
|
});
|
|
});
|
|
|
|
// ==========================================================================
|
|
// get (cache hit)
|
|
// ==========================================================================
|
|
|
|
describe('get (hit)', () => {
|
|
it('should return cached entry on hit', () => {
|
|
const fp = 'test-fingerprint';
|
|
cache.set(fp, 'domain', 'action', { answer: 42 }, 150);
|
|
|
|
const result = cache.get(fp);
|
|
expect(result).not.toBeNull();
|
|
expect(result!.fingerprint).toBe(fp);
|
|
expect(result!.domain).toBe('domain');
|
|
expect(result!.action).toBe('action');
|
|
expect(result!.result).toEqual({ answer: 42 });
|
|
expect(result!.tokensSaved).toBe(150);
|
|
});
|
|
|
|
it('should increment hitCount on each hit', () => {
|
|
const fp = 'test-fp';
|
|
cache.set(fp, 'd', 'a', {}, 100);
|
|
|
|
cache.get(fp);
|
|
cache.get(fp);
|
|
const entry = cache.get(fp);
|
|
expect(entry!.hitCount).toBe(3);
|
|
});
|
|
|
|
it('should update lastHitAt on hit', () => {
|
|
const fp = 'test-fp';
|
|
cache.set(fp, 'd', 'a', {}, 100);
|
|
|
|
const before = Date.now();
|
|
const entry = cache.get(fp);
|
|
expect(entry!.lastHitAt).toBeGreaterThanOrEqual(before);
|
|
});
|
|
|
|
it('should increment hits counter', () => {
|
|
cache.set('fp1', 'd', 'a', {}, 100);
|
|
cache.get('fp1');
|
|
cache.get('fp1');
|
|
const stats = cache.getStats();
|
|
expect(stats.hits).toBe(2);
|
|
});
|
|
});
|
|
|
|
// ==========================================================================
|
|
// TTL expiry
|
|
// ==========================================================================
|
|
|
|
describe('TTL expiry', () => {
|
|
it('should return null for expired entries', () => {
|
|
// Create cache with very short TTL
|
|
const shortTtlCache = new SessionOperationCache({
|
|
ttlMs: 1,
|
|
persistToDb: false,
|
|
});
|
|
|
|
shortTtlCache.set('fp', 'd', 'a', {}, 100);
|
|
|
|
// Wait a tiny bit to ensure TTL expires
|
|
// The entry cachedAt is set to Date.now() in set(), and TTL is 1ms
|
|
// We need to advance past the TTL
|
|
vi.useFakeTimers();
|
|
vi.advanceTimersByTime(5);
|
|
|
|
const result = shortTtlCache.get('fp');
|
|
expect(result).toBeNull();
|
|
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
it('should count expired lookups as misses', () => {
|
|
const shortTtlCache = new SessionOperationCache({
|
|
ttlMs: 1,
|
|
persistToDb: false,
|
|
});
|
|
|
|
shortTtlCache.set('fp', 'd', 'a', {}, 100);
|
|
|
|
vi.useFakeTimers();
|
|
vi.advanceTimersByTime(5);
|
|
|
|
shortTtlCache.get('fp');
|
|
const stats = shortTtlCache.getStats();
|
|
expect(stats.misses).toBe(1);
|
|
expect(stats.hits).toBe(0);
|
|
|
|
vi.useRealTimers();
|
|
});
|
|
});
|
|
|
|
// ==========================================================================
|
|
// Eviction
|
|
// ==========================================================================
|
|
|
|
describe('eviction', () => {
|
|
it('should evict oldest entry when at maxEntries', () => {
|
|
const smallCache = new SessionOperationCache({
|
|
maxEntries: 2,
|
|
persistToDb: false,
|
|
});
|
|
|
|
smallCache.set('fp1', 'd', 'a', { v: 1 }, 100);
|
|
smallCache.set('fp2', 'd', 'a', { v: 2 }, 200);
|
|
// This should evict fp1 (oldest)
|
|
smallCache.set('fp3', 'd', 'a', { v: 3 }, 300);
|
|
|
|
expect(smallCache.get('fp1')).toBeNull();
|
|
expect(smallCache.get('fp3')).not.toBeNull();
|
|
// fp2 should still be there
|
|
expect(smallCache.get('fp2')).not.toBeNull();
|
|
});
|
|
});
|
|
|
|
// ==========================================================================
|
|
// getStats
|
|
// ==========================================================================
|
|
|
|
describe('getStats', () => {
|
|
it('should return correct hit rate', () => {
|
|
cache.set('fp1', 'd', 'a', {}, 100);
|
|
|
|
cache.get('fp1'); // hit
|
|
cache.get('fp1'); // hit
|
|
cache.get('missing'); // miss
|
|
|
|
const stats = cache.getStats();
|
|
expect(stats.hits).toBe(2);
|
|
expect(stats.misses).toBe(1);
|
|
expect(stats.hitRate).toBeCloseTo(2 / 3);
|
|
});
|
|
|
|
it('should return correct estimated tokens saved', () => {
|
|
cache.set('fp1', 'd', 'a', {}, 100);
|
|
cache.set('fp2', 'd', 'a', {}, 200);
|
|
|
|
cache.get('fp1'); // hit, 100 tokens saved once
|
|
cache.get('fp1'); // hit, 100 tokens saved again
|
|
cache.get('fp2'); // hit, 200 tokens saved once
|
|
|
|
const stats = cache.getStats();
|
|
// fp1: 100 * 2 hits = 200, fp2: 200 * 1 hit = 200 => total 400
|
|
expect(stats.estimatedTokensSaved).toBe(400);
|
|
});
|
|
|
|
it('should return hitRate 0 with no lookups', () => {
|
|
const stats = cache.getStats();
|
|
expect(stats.hitRate).toBe(0);
|
|
});
|
|
|
|
it('should return correct size', () => {
|
|
cache.set('fp1', 'd', 'a', {}, 100);
|
|
cache.set('fp2', 'd', 'a', {}, 200);
|
|
expect(cache.getStats().size).toBe(2);
|
|
});
|
|
});
|
|
|
|
// ==========================================================================
|
|
// clear
|
|
// ==========================================================================
|
|
|
|
describe('clear', () => {
|
|
it('should clear all entries and reset counters', () => {
|
|
cache.set('fp1', 'd', 'a', {}, 100);
|
|
cache.get('fp1');
|
|
cache.get('missing');
|
|
|
|
cache.clear();
|
|
|
|
const stats = cache.getStats();
|
|
expect(stats.size).toBe(0);
|
|
expect(stats.hits).toBe(0);
|
|
expect(stats.misses).toBe(0);
|
|
expect(stats.estimatedTokensSaved).toBe(0);
|
|
});
|
|
|
|
it('should not return previously cached entries after clear', () => {
|
|
cache.set('fp1', 'd', 'a', {}, 100);
|
|
cache.clear();
|
|
expect(cache.get('fp1')).toBeNull();
|
|
});
|
|
});
|
|
|
|
// ==========================================================================
|
|
// Disabled cache
|
|
// ==========================================================================
|
|
|
|
describe('disabled cache', () => {
|
|
it('should always return null from get when disabled', () => {
|
|
const disabled = new SessionOperationCache({
|
|
enabled: false,
|
|
persistToDb: false,
|
|
});
|
|
|
|
disabled.set('fp1', 'd', 'a', {}, 100);
|
|
expect(disabled.get('fp1')).toBeNull();
|
|
});
|
|
|
|
it('should not store entries when disabled', () => {
|
|
const disabled = new SessionOperationCache({
|
|
enabled: false,
|
|
persistToDb: false,
|
|
});
|
|
|
|
disabled.set('fp1', 'd', 'a', {}, 100);
|
|
expect(disabled.getStats().size).toBe(0);
|
|
});
|
|
});
|
|
|
|
// ==========================================================================
|
|
// loadFromDb graceful degradation
|
|
// ==========================================================================
|
|
|
|
describe('loadFromDb', () => {
|
|
it('should gracefully degrade when DB is unavailable', () => {
|
|
// By default, the DB is not initialized in tests.
|
|
// loadFromDb should not throw.
|
|
expect(() => cache.loadFromDb()).not.toThrow();
|
|
expect(cache.getStats().size).toBe(0);
|
|
});
|
|
});
|
|
|
|
// ==========================================================================
|
|
// Singleton
|
|
// ==========================================================================
|
|
|
|
describe('getSessionCache singleton', () => {
|
|
it('should return the same instance on repeated calls', () => {
|
|
const a = getSessionCache({ persistToDb: false });
|
|
const b = getSessionCache();
|
|
expect(a).toBe(b);
|
|
});
|
|
|
|
it('should return a fresh instance after resetSessionCache', () => {
|
|
const a = getSessionCache({ persistToDb: false });
|
|
resetSessionCache();
|
|
const b = getSessionCache({ persistToDb: false });
|
|
expect(a).not.toBe(b);
|
|
});
|
|
});
|
|
|
|
// ==========================================================================
|
|
// Default config
|
|
// ==========================================================================
|
|
|
|
describe('default config', () => {
|
|
it('should have sensible defaults', () => {
|
|
expect(DEFAULT_SESSION_CACHE_CONFIG.enabled).toBe(true);
|
|
expect(DEFAULT_SESSION_CACHE_CONFIG.maxEntries).toBe(500);
|
|
expect(DEFAULT_SESSION_CACHE_CONFIG.ttlMs).toBe(60 * 60 * 1000);
|
|
expect(DEFAULT_SESSION_CACHE_CONFIG.persistToDb).toBe(true);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('#569 follow-up — cache keys must not outlive the build that made them', () => {
|
|
afterEach(() => {
|
|
delete process.env.AQE_SESSION_CACHE_SALT;
|
|
});
|
|
|
|
it('changes the fingerprint when the build identity changes', () => {
|
|
// Entries persist to kv_store and survive package upgrades. Keyed on inputs
|
|
// alone, an upgraded install replays the PREVIOUS version's answers until
|
|
// the TTL lapses. That is exactly what happened while verifying #569: a
|
|
// project analyzed before the fix kept returning the old fabricated
|
|
// coverage (lineCoverage: 0, measured: true) from cache, making the fix
|
|
// look broken.
|
|
const cache = new SessionOperationCache();
|
|
const input = { target: '.', detectGaps: true };
|
|
|
|
const before = cache.computeFingerprint('coverage', 'analyze_sublinear', input);
|
|
process.env.AQE_SESSION_CACHE_SALT = 'next-build';
|
|
const after = cache.computeFingerprint('coverage', 'analyze_sublinear', input);
|
|
|
|
expect(after).not.toBe(before);
|
|
});
|
|
|
|
it('is still deterministic for identical inputs within one build', () => {
|
|
// Guard against over-correction: the cache must still actually cache.
|
|
const cache = new SessionOperationCache();
|
|
const input = { target: '.', detectGaps: true };
|
|
expect(cache.computeFingerprint('coverage', 'analyze_sublinear', input))
|
|
.toBe(cache.computeFingerprint('coverage', 'analyze_sublinear', input));
|
|
});
|
|
|
|
it('still distinguishes different inputs', () => {
|
|
const cache = new SessionOperationCache();
|
|
expect(cache.computeFingerprint('coverage', 'analyze_sublinear', { target: 'a' }))
|
|
.not.toBe(cache.computeFingerprint('coverage', 'analyze_sublinear', { target: 'b' }));
|
|
});
|
|
});
|