mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Dragan Spiridonov
e266bae929
Before this commit two parallel writers bumped pattern usage stats:
- SQLitePatternStore.recordUsage — did BOTH qe_pattern_usage INSERT
and qe_patterns UPDATE inside one transaction (correct).
- hooks-dream-learning.ts inline UPDATE — bumped qe_patterns columns
but SKIPPED the qe_pattern_usage INSERT, so the audit table stayed
EMPTY in production even as qe_patterns.usage_count climbed during
normal hook activity. V0.03 evidence: 0 rows in qe_pattern_usage
despite 33 hits on the dominant cli-hook-code-intelligence pattern.
Extracts the shared accounting into a new helper
src/learning/pattern-usage-recorder.ts with a single transaction:
- INSERT INTO qe_pattern_usage (audit trail per use)
- UPDATE qe_patterns.{usage_count, successful_uses, success_rate,
quality_score, last_used_at, updated_at}
Both writes are atomic — either both land or neither does.
Quality formula matches the canonical pattern-lifecycle.ts shape:
quality_score = confidence*0.3 + min(usage_count/100, 1)*0.2 + success_rate*0.5
Returns updated=false (no throw) when patternId is unknown, because hook
subprocesses iterate selectedPatternIds that may contain stale UUIDs
from a routing kv that drifted from qe_patterns. Callers wanting strict
semantics (SQLitePatternStore.recordUsage) check the flag and throw.
Refactors:
- SQLitePatternStore.recordUsage → thin wrapper around the helper.
Preserves the throw-on-missing contract.
- hooks-dream-learning.ts per-pattern inline UPDATE → replaced with
recordPatternUsage() call. Now passes meaningful metrics
(tokens_saved, source, experience_id) and feedback to the audit row.
Keeps the #455 inline promotion check (separate concern, fires
AFTER the helper has updated the qe_patterns columns).
Invariant established: COUNT(*) FROM qe_pattern_usage WHERE pattern_id=X
equals qe_patterns.usage_count for X. Before this commit that invariant
was systematically violated by the hook path.
Test infrastructure:
- 7 new tests in tests/unit/learning/pattern-usage-recorder.test.ts
cover atomicity, the missing-pattern no-throw contract, the formula
boundary at usage_count=100, null metrics/feedback handling, and
the audit/column count invariant.
- tests/unit/cli/commands/post-task-promotion.test.ts schema seeds
now include qe_pattern_usage (the test fixture was missing the
table that the production schema creates at boot).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
301 lines
8.6 KiB
TypeScript
301 lines
8.6 KiB
TypeScript
/**
|
|
* Regression: issue #455
|
|
*
|
|
* The post-task bridge loop updates qe_patterns.{usage_count, successful_uses,
|
|
* success_rate, quality_score} for real pattern UUIDs but never promoted
|
|
* short-term → long-term. recordOutcome() runs with a synthetic
|
|
* `task:agent:taskId` patternId that never matches qe_patterns.id, so
|
|
* checkPatternPromotionWithCoherence() was skipped for the real UUIDs.
|
|
*
|
|
* Patterns with successful_uses ≥ 3, success_rate ≥ 0.7, confidence ≥ 0.6
|
|
* accumulated forever at tier='short-term'. This test seeds a row one
|
|
* success-away from the promotion threshold and asserts the bridge loop
|
|
* flips its tier to 'long-term' after a successful post-task invocation.
|
|
*/
|
|
import path from 'node:path';
|
|
import fs from 'node:fs';
|
|
import os from 'node:os';
|
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
|
|
|
const { fakeUnifiedMemoryFactory } = vi.hoisted(() => ({
|
|
fakeUnifiedMemoryFactory: vi.fn(),
|
|
}));
|
|
|
|
vi.mock('../../../../src/kernel/unified-memory.js', () => ({
|
|
getUnifiedMemory: () => fakeUnifiedMemoryFactory(),
|
|
findProjectRoot: () => process.cwd(),
|
|
}));
|
|
|
|
import Database from 'better-sqlite3';
|
|
import { persistTaskOutcome } from '../../../../src/cli/commands/hooks-handlers/hooks-dream-learning.js';
|
|
|
|
describe('post-task bridge loop promotes patterns (#455)', () => {
|
|
let tmpDir: string;
|
|
let db: Database.Database;
|
|
|
|
const createSchema = (database: Database.Database): void => {
|
|
database.exec(`
|
|
CREATE TABLE captured_experiences (
|
|
id TEXT PRIMARY KEY,
|
|
task TEXT NOT NULL,
|
|
agent TEXT,
|
|
domain TEXT,
|
|
success INTEGER,
|
|
quality REAL,
|
|
duration_ms INTEGER,
|
|
model_tier TEXT,
|
|
started_at TEXT,
|
|
completed_at TEXT,
|
|
source TEXT,
|
|
consolidated_into TEXT
|
|
);
|
|
CREATE TABLE experience_applications (
|
|
id TEXT PRIMARY KEY,
|
|
experience_id TEXT,
|
|
task TEXT,
|
|
success INTEGER,
|
|
tokens_saved INTEGER,
|
|
feedback TEXT,
|
|
applied_at TEXT
|
|
);
|
|
CREATE TABLE qe_trajectories (
|
|
id TEXT PRIMARY KEY,
|
|
task TEXT,
|
|
agent TEXT,
|
|
domain TEXT,
|
|
started_at TEXT,
|
|
ended_at TEXT,
|
|
success INTEGER,
|
|
steps_json TEXT
|
|
);
|
|
CREATE TABLE qe_patterns (
|
|
id TEXT PRIMARY KEY,
|
|
pattern_type TEXT,
|
|
qe_domain TEXT,
|
|
domain TEXT,
|
|
name TEXT,
|
|
description TEXT,
|
|
confidence REAL DEFAULT 0.5,
|
|
usage_count INTEGER DEFAULT 0,
|
|
successful_uses INTEGER DEFAULT 0,
|
|
success_rate REAL DEFAULT 0.0,
|
|
quality_score REAL DEFAULT 0.0,
|
|
tier TEXT DEFAULT 'short-term',
|
|
last_used_at TEXT,
|
|
created_at TEXT,
|
|
updated_at TEXT
|
|
);
|
|
-- #486 Gap B: hook path now writes the audit row alongside the qe_patterns
|
|
-- UPDATE in a single transaction. Without this table the helper's INSERT
|
|
-- would throw and the transaction would roll back, blocking the UPDATE.
|
|
CREATE TABLE qe_pattern_usage (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
pattern_id TEXT NOT NULL,
|
|
success INTEGER NOT NULL,
|
|
metrics_json TEXT,
|
|
feedback TEXT,
|
|
recorded_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE kv_store (
|
|
key TEXT,
|
|
namespace TEXT,
|
|
value TEXT,
|
|
expires_at INTEGER,
|
|
created_at INTEGER,
|
|
PRIMARY KEY (namespace, key)
|
|
);
|
|
CREATE TABLE dream_insights (
|
|
id TEXT PRIMARY KEY,
|
|
applied INTEGER DEFAULT 0,
|
|
actionable INTEGER DEFAULT 0,
|
|
created_at TEXT
|
|
);
|
|
`);
|
|
};
|
|
|
|
beforeEach(() => {
|
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'aqe-promotion-'));
|
|
db = new Database(path.join(tmpDir, 'unified.db'));
|
|
createSchema(db);
|
|
|
|
fakeUnifiedMemoryFactory.mockReturnValue({
|
|
isInitialized: () => true,
|
|
initialize: async () => undefined,
|
|
getDatabase: () => db,
|
|
});
|
|
});
|
|
|
|
afterEach(() => {
|
|
try {
|
|
db.close();
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
try {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
});
|
|
|
|
const seedBridge = (patternId: string): void => {
|
|
db.prepare(`
|
|
INSERT INTO kv_store (key, namespace, value, expires_at, created_at)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
`).run(
|
|
'task:abc',
|
|
'task-bridge',
|
|
JSON.stringify({
|
|
selectedPatternIds: [patternId],
|
|
agent: 'qe-test-architect',
|
|
description: 'd',
|
|
taskType: 'test-generation',
|
|
priority: 'normal',
|
|
domain: 'general',
|
|
complexityBucket: 3,
|
|
estimatedTokenSavings: 0,
|
|
ts: Date.now(),
|
|
}),
|
|
Date.now() + 600_000,
|
|
Date.now(),
|
|
);
|
|
};
|
|
|
|
it('promotes a short-term pattern to long-term when thresholds are met', async () => {
|
|
// Seed a pattern one success away from promotion. After bridge update:
|
|
// usage_count: 2 → 3
|
|
// successful_uses: 2 → 3 (PROMOTION_THRESHOLD)
|
|
// success_rate: 2/2=1.0 → 3/3=1.0 (≥ 0.7)
|
|
// confidence: 0.7 (≥ 0.6)
|
|
db.prepare(`
|
|
INSERT INTO qe_patterns
|
|
(id, pattern_type, qe_domain, domain, name,
|
|
confidence, usage_count, successful_uses, success_rate, tier)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
`).run(
|
|
'pat-ready',
|
|
'test-template',
|
|
'test-generation',
|
|
'test-generation',
|
|
'Ready Pattern',
|
|
0.7,
|
|
2,
|
|
2,
|
|
1.0,
|
|
'short-term',
|
|
);
|
|
seedBridge('pat-ready');
|
|
|
|
await persistTaskOutcome({
|
|
taskId: 'hook-test',
|
|
agent: 'qe-test-architect',
|
|
success: true,
|
|
});
|
|
|
|
const row = db
|
|
.prepare(`SELECT tier, successful_uses, success_rate, confidence FROM qe_patterns WHERE id = ?`)
|
|
.get('pat-ready') as { tier: string; successful_uses: number; success_rate: number; confidence: number };
|
|
|
|
expect(row.successful_uses).toBe(3);
|
|
expect(row.success_rate).toBeCloseTo(1.0, 5);
|
|
expect(row.tier).toBe('long-term');
|
|
});
|
|
|
|
it('does NOT promote when success_rate is below threshold', async () => {
|
|
db.prepare(`
|
|
INSERT INTO qe_patterns
|
|
(id, pattern_type, qe_domain, domain, name,
|
|
confidence, usage_count, successful_uses, success_rate, tier)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
`).run(
|
|
'pat-low-success',
|
|
'test-template',
|
|
'test-generation',
|
|
'test-generation',
|
|
'Low Success Pattern',
|
|
0.7,
|
|
5, // usage_count = 5
|
|
3, // successful_uses = 3 (meets threshold)
|
|
0.6, // success_rate = 0.6 (BELOW 0.7 threshold)
|
|
'short-term',
|
|
);
|
|
seedBridge('pat-low-success');
|
|
|
|
await persistTaskOutcome({
|
|
taskId: 'hook-test',
|
|
agent: 'qe-test-architect',
|
|
success: false, // failure — drops success_rate further
|
|
});
|
|
|
|
const row = db
|
|
.prepare(`SELECT tier FROM qe_patterns WHERE id = ?`)
|
|
.get('pat-low-success') as { tier: string };
|
|
expect(row.tier).toBe('short-term');
|
|
});
|
|
|
|
it('does NOT promote when confidence is below threshold', async () => {
|
|
db.prepare(`
|
|
INSERT INTO qe_patterns
|
|
(id, pattern_type, qe_domain, domain, name,
|
|
confidence, usage_count, successful_uses, success_rate, tier)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
`).run(
|
|
'pat-low-conf',
|
|
'test-template',
|
|
'test-generation',
|
|
'test-generation',
|
|
'Low Conf Pattern',
|
|
0.4, // confidence BELOW 0.6 threshold
|
|
2,
|
|
2,
|
|
1.0,
|
|
'short-term',
|
|
);
|
|
seedBridge('pat-low-conf');
|
|
|
|
await persistTaskOutcome({
|
|
taskId: 'hook-test',
|
|
agent: 'qe-test-architect',
|
|
success: true,
|
|
});
|
|
|
|
const row = db
|
|
.prepare(`SELECT tier FROM qe_patterns WHERE id = ?`)
|
|
.get('pat-low-conf') as { tier: string };
|
|
expect(row.tier).toBe('short-term');
|
|
});
|
|
|
|
it('leaves already-long-term rows untouched (idempotent)', async () => {
|
|
db.prepare(`
|
|
INSERT INTO qe_patterns
|
|
(id, pattern_type, qe_domain, domain, name,
|
|
confidence, usage_count, successful_uses, success_rate, tier, updated_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
`).run(
|
|
'pat-already-long',
|
|
'test-template',
|
|
'test-generation',
|
|
'test-generation',
|
|
'Already Long Pattern',
|
|
0.9,
|
|
10,
|
|
9,
|
|
0.9,
|
|
'long-term',
|
|
'2026-01-01 00:00:00',
|
|
);
|
|
seedBridge('pat-already-long');
|
|
|
|
await persistTaskOutcome({
|
|
taskId: 'hook-test',
|
|
agent: 'qe-test-architect',
|
|
success: true,
|
|
});
|
|
|
|
const row = db
|
|
.prepare(`SELECT tier FROM qe_patterns WHERE id = ?`)
|
|
.get('pat-already-long') as { tier: string };
|
|
expect(row.tier).toBe('long-term');
|
|
});
|
|
});
|