Files
Dragan Spiridonov b6fd679254 fix(#569): stop the MCP layer laundering missing provenance into confident numbers
Found by the qe-devils-advocate session review, AFTER #569 was already committed.
The handler was producing honest provenance; `coverageAnalyzeConfig.mapToResult`
was converting it back into fabricated certainty on three paths.

1. Absent provenance was treated as "collected".
   `data.branchDataCollected !== false` — and `undefined !== false` is true. Any
   producer that supplies no provenance (notably the coverage-analysis domain
   plugin, whose handler returns a nested-`summary` CoverageReport) therefore
   surfaced issue #569's EXACT reported symptom: 100% branch coverage alongside
   0% function coverage, labelled `measured: true`, with `confidence: 0.7` on its
   gaps. Now requires an explicit `true`; absence means unknown, so an
   unprovenanced result reports `null`.

2. `measured` was derived as `!estimated`.
   These are not complements — there is a third state where nothing was collected
   at all. An empty project reported `lineCoverage: 0, measured: true`: "we
   measured 0%" when the truth was "we measured nothing". Verified through the
   real MCP server before and after. `measured` now requires an explicit claim.

3. `null` was coerced to `0`, and risk was graded off it.
   `?? 0` turned "no data" into a plottable 0%, and `null < 70` is `false` in JS,
   so an unguarded comparison graded "no data" as 'low' risk — the most dangerous
   possible default. Nulls now survive; risk grading requires a real number.

Also fixes the stale-result mystery that dogged verification all session.
`src/optimization/session-cache.ts` fingerprints on domain+action+input only, and
persists to kv_store with a 1-hour TTL, so entries survive package upgrades: a
project analyzed before a fix keeps replaying the old answer, making the fix look
broken. I twice mis-diagnosed this (first as a cache, then wrongly retracted that;
it WAS the cache). The fingerprint now includes the build identity, so an upgrade
misses cleanly and old entries age out. `AQE_SESSION_CACHE_SALT` forces a cold
cache without deleting anything.

This matters beyond #569: every fix in this release would have appeared not to
work for any user with a warm cache.

`CoverageAnalyzeResult.lineCoverage`/`statementCoverage` are now `number | null`,
joining `branchCoverage`/`functionCoverage`. `0` means a measured zero; `null`
means unknown.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-23 14:46:31 +00:00

372 lines
12 KiB
TypeScript

/**
* Unit Tests: SessionOperationCache
* Imp-15: Session Reuse for Repeated Operations
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import {
SessionOperationCache,
getSessionCache,
resetSessionCache,
DEFAULT_SESSION_CACHE_CONFIG,
} from '../../../src/optimization/session-cache.js';
describe('SessionOperationCache', () => {
let cache: SessionOperationCache;
beforeEach(() => {
resetSessionCache();
cache = new SessionOperationCache({ persistToDb: false });
});
afterEach(() => {
vi.restoreAllMocks();
});
// ==========================================================================
// computeFingerprint
// ==========================================================================
describe('computeFingerprint', () => {
it('should produce consistent hashes for the same input', () => {
const fp1 = cache.computeFingerprint('test-gen', 'generate', { file: 'a.ts' });
const fp2 = cache.computeFingerprint('test-gen', 'generate', { file: 'a.ts' });
expect(fp1).toBe(fp2);
});
it('should produce different hashes for different inputs', () => {
const fp1 = cache.computeFingerprint('test-gen', 'generate', { file: 'a.ts' });
const fp2 = cache.computeFingerprint('test-gen', 'generate', { file: 'b.ts' });
expect(fp1).not.toBe(fp2);
});
it('should produce different hashes for different domains', () => {
const fp1 = cache.computeFingerprint('domain-a', 'action', {});
const fp2 = cache.computeFingerprint('domain-b', 'action', {});
expect(fp1).not.toBe(fp2);
});
it('should produce different hashes for different actions', () => {
const fp1 = cache.computeFingerprint('domain', 'action-a', {});
const fp2 = cache.computeFingerprint('domain', 'action-b', {});
expect(fp1).not.toBe(fp2);
});
it('should return a 16-character hex string', () => {
const fp = cache.computeFingerprint('d', 'a', { x: 1 });
expect(fp).toMatch(/^[0-9a-f]{16}$/);
});
});
// ==========================================================================
// get (cache miss)
// ==========================================================================
describe('get (miss)', () => {
it('should return null on miss', () => {
const result = cache.get('nonexistent');
expect(result).toBeNull();
});
it('should increment misses counter on miss', () => {
cache.get('nonexistent');
cache.get('also-nonexistent');
const stats = cache.getStats();
expect(stats.misses).toBe(2);
expect(stats.hits).toBe(0);
});
});
// ==========================================================================
// get (cache hit)
// ==========================================================================
describe('get (hit)', () => {
it('should return cached entry on hit', () => {
const fp = 'test-fingerprint';
cache.set(fp, 'domain', 'action', { answer: 42 }, 150);
const result = cache.get(fp);
expect(result).not.toBeNull();
expect(result!.fingerprint).toBe(fp);
expect(result!.domain).toBe('domain');
expect(result!.action).toBe('action');
expect(result!.result).toEqual({ answer: 42 });
expect(result!.tokensSaved).toBe(150);
});
it('should increment hitCount on each hit', () => {
const fp = 'test-fp';
cache.set(fp, 'd', 'a', {}, 100);
cache.get(fp);
cache.get(fp);
const entry = cache.get(fp);
expect(entry!.hitCount).toBe(3);
});
it('should update lastHitAt on hit', () => {
const fp = 'test-fp';
cache.set(fp, 'd', 'a', {}, 100);
const before = Date.now();
const entry = cache.get(fp);
expect(entry!.lastHitAt).toBeGreaterThanOrEqual(before);
});
it('should increment hits counter', () => {
cache.set('fp1', 'd', 'a', {}, 100);
cache.get('fp1');
cache.get('fp1');
const stats = cache.getStats();
expect(stats.hits).toBe(2);
});
});
// ==========================================================================
// TTL expiry
// ==========================================================================
describe('TTL expiry', () => {
it('should return null for expired entries', () => {
// Create cache with very short TTL
const shortTtlCache = new SessionOperationCache({
ttlMs: 1,
persistToDb: false,
});
shortTtlCache.set('fp', 'd', 'a', {}, 100);
// Wait a tiny bit to ensure TTL expires
// The entry cachedAt is set to Date.now() in set(), and TTL is 1ms
// We need to advance past the TTL
vi.useFakeTimers();
vi.advanceTimersByTime(5);
const result = shortTtlCache.get('fp');
expect(result).toBeNull();
vi.useRealTimers();
});
it('should count expired lookups as misses', () => {
const shortTtlCache = new SessionOperationCache({
ttlMs: 1,
persistToDb: false,
});
shortTtlCache.set('fp', 'd', 'a', {}, 100);
vi.useFakeTimers();
vi.advanceTimersByTime(5);
shortTtlCache.get('fp');
const stats = shortTtlCache.getStats();
expect(stats.misses).toBe(1);
expect(stats.hits).toBe(0);
vi.useRealTimers();
});
});
// ==========================================================================
// Eviction
// ==========================================================================
describe('eviction', () => {
it('should evict oldest entry when at maxEntries', () => {
const smallCache = new SessionOperationCache({
maxEntries: 2,
persistToDb: false,
});
smallCache.set('fp1', 'd', 'a', { v: 1 }, 100);
smallCache.set('fp2', 'd', 'a', { v: 2 }, 200);
// This should evict fp1 (oldest)
smallCache.set('fp3', 'd', 'a', { v: 3 }, 300);
expect(smallCache.get('fp1')).toBeNull();
expect(smallCache.get('fp3')).not.toBeNull();
// fp2 should still be there
expect(smallCache.get('fp2')).not.toBeNull();
});
});
// ==========================================================================
// getStats
// ==========================================================================
describe('getStats', () => {
it('should return correct hit rate', () => {
cache.set('fp1', 'd', 'a', {}, 100);
cache.get('fp1'); // hit
cache.get('fp1'); // hit
cache.get('missing'); // miss
const stats = cache.getStats();
expect(stats.hits).toBe(2);
expect(stats.misses).toBe(1);
expect(stats.hitRate).toBeCloseTo(2 / 3);
});
it('should return correct estimated tokens saved', () => {
cache.set('fp1', 'd', 'a', {}, 100);
cache.set('fp2', 'd', 'a', {}, 200);
cache.get('fp1'); // hit, 100 tokens saved once
cache.get('fp1'); // hit, 100 tokens saved again
cache.get('fp2'); // hit, 200 tokens saved once
const stats = cache.getStats();
// fp1: 100 * 2 hits = 200, fp2: 200 * 1 hit = 200 => total 400
expect(stats.estimatedTokensSaved).toBe(400);
});
it('should return hitRate 0 with no lookups', () => {
const stats = cache.getStats();
expect(stats.hitRate).toBe(0);
});
it('should return correct size', () => {
cache.set('fp1', 'd', 'a', {}, 100);
cache.set('fp2', 'd', 'a', {}, 200);
expect(cache.getStats().size).toBe(2);
});
});
// ==========================================================================
// clear
// ==========================================================================
describe('clear', () => {
it('should clear all entries and reset counters', () => {
cache.set('fp1', 'd', 'a', {}, 100);
cache.get('fp1');
cache.get('missing');
cache.clear();
const stats = cache.getStats();
expect(stats.size).toBe(0);
expect(stats.hits).toBe(0);
expect(stats.misses).toBe(0);
expect(stats.estimatedTokensSaved).toBe(0);
});
it('should not return previously cached entries after clear', () => {
cache.set('fp1', 'd', 'a', {}, 100);
cache.clear();
expect(cache.get('fp1')).toBeNull();
});
});
// ==========================================================================
// Disabled cache
// ==========================================================================
describe('disabled cache', () => {
it('should always return null from get when disabled', () => {
const disabled = new SessionOperationCache({
enabled: false,
persistToDb: false,
});
disabled.set('fp1', 'd', 'a', {}, 100);
expect(disabled.get('fp1')).toBeNull();
});
it('should not store entries when disabled', () => {
const disabled = new SessionOperationCache({
enabled: false,
persistToDb: false,
});
disabled.set('fp1', 'd', 'a', {}, 100);
expect(disabled.getStats().size).toBe(0);
});
});
// ==========================================================================
// loadFromDb graceful degradation
// ==========================================================================
describe('loadFromDb', () => {
it('should gracefully degrade when DB is unavailable', () => {
// By default, the DB is not initialized in tests.
// loadFromDb should not throw.
expect(() => cache.loadFromDb()).not.toThrow();
expect(cache.getStats().size).toBe(0);
});
});
// ==========================================================================
// Singleton
// ==========================================================================
describe('getSessionCache singleton', () => {
it('should return the same instance on repeated calls', () => {
const a = getSessionCache({ persistToDb: false });
const b = getSessionCache();
expect(a).toBe(b);
});
it('should return a fresh instance after resetSessionCache', () => {
const a = getSessionCache({ persistToDb: false });
resetSessionCache();
const b = getSessionCache({ persistToDb: false });
expect(a).not.toBe(b);
});
});
// ==========================================================================
// Default config
// ==========================================================================
describe('default config', () => {
it('should have sensible defaults', () => {
expect(DEFAULT_SESSION_CACHE_CONFIG.enabled).toBe(true);
expect(DEFAULT_SESSION_CACHE_CONFIG.maxEntries).toBe(500);
expect(DEFAULT_SESSION_CACHE_CONFIG.ttlMs).toBe(60 * 60 * 1000);
expect(DEFAULT_SESSION_CACHE_CONFIG.persistToDb).toBe(true);
});
});
});
describe('#569 follow-up — cache keys must not outlive the build that made them', () => {
afterEach(() => {
delete process.env.AQE_SESSION_CACHE_SALT;
});
it('changes the fingerprint when the build identity changes', () => {
// Entries persist to kv_store and survive package upgrades. Keyed on inputs
// alone, an upgraded install replays the PREVIOUS version's answers until
// the TTL lapses. That is exactly what happened while verifying #569: a
// project analyzed before the fix kept returning the old fabricated
// coverage (lineCoverage: 0, measured: true) from cache, making the fix
// look broken.
const cache = new SessionOperationCache();
const input = { target: '.', detectGaps: true };
const before = cache.computeFingerprint('coverage', 'analyze_sublinear', input);
process.env.AQE_SESSION_CACHE_SALT = 'next-build';
const after = cache.computeFingerprint('coverage', 'analyze_sublinear', input);
expect(after).not.toBe(before);
});
it('is still deterministic for identical inputs within one build', () => {
// Guard against over-correction: the cache must still actually cache.
const cache = new SessionOperationCache();
const input = { target: '.', detectGaps: true };
expect(cache.computeFingerprint('coverage', 'analyze_sublinear', input))
.toBe(cache.computeFingerprint('coverage', 'analyze_sublinear', input));
});
it('still distinguishes different inputs', () => {
const cache = new SessionOperationCache();
expect(cache.computeFingerprint('coverage', 'analyze_sublinear', { target: 'a' }))
.not.toBe(cache.computeFingerprint('coverage', 'analyze_sublinear', { target: 'b' }));
});
});