mirror of
https://github.com/rohitg00/agentmemory.git
synced 2026-09-14 20:16:33 +08:00
7fb72f4010
* feat(eval): pluggable benchmark harness with in-house coding-agent corpus Adds eval/ tree (outside files field so npm tarball stays thin) with Adapter interface, three reference adapters (grep / vector / agentmemory-hybrid), two benchmarks (LongMemEval _s public, coding-agent-life-v1 in-house 15 sessions), scoring (P@K, R@K, hit, top-gold-rank), NDJSON output, sandbox script. coding-agent-life-v1 published scorecard at docs/benchmarks/2026-05-20-coding-agent-life-v1.md: agentmemory-hybrid R@5=0.967 P@5=0.578 (100% hit) vs grep R@5=0.967 P@5=0.267. 2.2x better precision on identical input, sandbox-reproducible. Adapter contract: init(sessions, config) -> State; query(q, state, k) -> RankedDoc[] npm scripts: npm run eval:coding-life (no download, no API key for grep) npm run eval:longmemeval (needs OPENAI key + 278MB download) eval/scripts/sandbox.sh boots clean agentmemory + iii-engine on ports 3411/3412 with isolated data dir; tears down on exit. README headline updated. 1072/1072 tests pass + 5 new eval tests. * fix(eval): address review findings on benchmark harness - agentmemory adapter: prefer row.sessionId before observationToSession lookup - vector adapter: validate embedBatch response (length, indexes, non-empty rows) - coding-life: positive-int guard on --k; wrap query loop in try/finally so teardown runs - longmemeval: positive-int guards on --k/--limit/--stratify; per-question try/finally - load: throw on haystack_session_ids vs haystack_sessions length mismatch - score: P@K denominator is k (requested cutoff) not topK.length - sandbox.sh: guard rm -rf with non-empty + /tmp/ prefix check - README: drop unsafe rm "$(which iii)"; instruct ~/.local/bin + PATH instead; add language tag to repo-layout fenced block - sessions.json: fix "two-phase" -> "three-phase" wording mismatch
94 lines
2.9 KiB
TypeScript
94 lines
2.9 KiB
TypeScript
import type { Adapter, RankedDoc, Session } from "../types.js";
|
|
|
|
interface AgentMemoryState {
|
|
baseUrl: string;
|
|
secret?: string;
|
|
sessions: Session[];
|
|
observationToSession: Map<string, string>;
|
|
}
|
|
|
|
interface RememberResponse {
|
|
memory?: { id?: string };
|
|
observationId?: string;
|
|
id?: string;
|
|
observation?: { id?: string };
|
|
}
|
|
|
|
interface SmartSearchResponse {
|
|
results?: Array<{
|
|
obsId?: string;
|
|
id?: string;
|
|
observationId?: string;
|
|
sessionId?: string;
|
|
score?: number;
|
|
content?: string;
|
|
}>;
|
|
observations?: Array<{
|
|
obsId?: string;
|
|
id?: string;
|
|
sessionId?: string;
|
|
score?: number;
|
|
content?: string;
|
|
}>;
|
|
}
|
|
|
|
function authHeaders(secret?: string): Record<string, string> {
|
|
const h: Record<string, string> = { "Content-Type": "application/json" };
|
|
if (secret) h.Authorization = `Bearer ${secret}`;
|
|
return h;
|
|
}
|
|
|
|
export const agentmemoryAdapter: Adapter<AgentMemoryState> = {
|
|
name: "agentmemory-hybrid",
|
|
async init(sessions, config) {
|
|
const baseUrl = (config?.baseUrl as string) ?? process.env.AGENTMEMORY_BASE_URL ?? "http://localhost:3111";
|
|
const secret = (config?.secret as string) ?? process.env.AGENTMEMORY_SECRET;
|
|
const observationToSession = new Map<string, string>();
|
|
for (const s of sessions) {
|
|
const res = await fetch(`${baseUrl}/agentmemory/remember`, {
|
|
method: "POST",
|
|
headers: authHeaders(secret),
|
|
body: JSON.stringify({
|
|
content: s.content,
|
|
type: "eval-session",
|
|
concepts: [s.id],
|
|
}),
|
|
});
|
|
if (!res.ok) {
|
|
throw new Error(`remember failed for ${s.id}: ${res.status} ${await res.text()}`);
|
|
}
|
|
const body = (await res.json()) as RememberResponse;
|
|
const obsId =
|
|
body.memory?.id ?? body.observationId ?? body.id ?? body.observation?.id;
|
|
if (obsId) observationToSession.set(obsId, s.id);
|
|
}
|
|
return { baseUrl, secret, sessions, observationToSession };
|
|
},
|
|
async query(q, state, k) {
|
|
const res = await fetch(`${state.baseUrl}/agentmemory/smart-search`, {
|
|
method: "POST",
|
|
headers: authHeaders(state.secret),
|
|
body: JSON.stringify({ query: q, limit: Math.max(k * 10, 50) }),
|
|
});
|
|
if (!res.ok) {
|
|
throw new Error(`smart-search failed: ${res.status} ${await res.text()}`);
|
|
}
|
|
const body = (await res.json()) as SmartSearchResponse;
|
|
const rows = body.results ?? body.observations ?? [];
|
|
const ranked: RankedDoc[] = [];
|
|
const seen = new Set<string>();
|
|
for (const row of rows) {
|
|
let sessionId = row.sessionId;
|
|
if (!sessionId) {
|
|
const memId = row.obsId ?? row.id ?? row.observationId;
|
|
sessionId = memId ? state.observationToSession.get(memId) : undefined;
|
|
}
|
|
if (!sessionId || seen.has(sessionId)) continue;
|
|
seen.add(sessionId);
|
|
ranked.push({ sessionId, score: row.score ?? 0 });
|
|
if (ranked.length >= k) break;
|
|
}
|
|
return ranked;
|
|
},
|
|
};
|