mirror of
https://github.com/colbymchenry/codegraph.git
synced 2026-09-19 07:34:57 +08:00
Colby McHenry
1b527aaf0f
Tooling to measure how a Claude Code agent actually uses the codegraph MCP tools on a real repo — does it lead with codegraph_explore, how many Read/Grep follow-ups, token cost — for validating tool-guidance changes (server-instructions, tool descriptions) against real agent behavior. - itrun.sh drives the real interactive TUI via tmux (the faithful Explore path). Hardened for unattended runs: type-and-verify prompt delivery (the ❯ glyph is drawn ~6s before the input accepts keys), auto-accepts the "trust this folder" dialog, busy-detection keys on the universal "(Ns · …)" spinner so the pre-stream thinking phase counts as busy, and fails loudly instead of capturing an empty pane. - parse-session.mjs reports the tool breakdown + token accounting (gen / fresh-in / cached-in / billable) from the session and subagent logs, consistent across main-thread and subagent runs; counts main-thread Bash in the grep verdict. - run-agent.sh / parse-run.mjs are the headless stream-json complement (exact per-tool tokens/cost via claude -p). - run-interactive-test.md documents how to run it and how completion is detected. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
46 lines
2.1 KiB
JavaScript
46 lines
2.1 KiB
JavaScript
#!/usr/bin/env node
|
|
// Parse a Claude Code stream-json run log: tool-call sequence + token usage.
|
|
import { readFileSync } from 'fs';
|
|
const file = process.argv[2];
|
|
const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
|
|
|
|
const toolCalls = [];
|
|
let result = null;
|
|
let initTools = null;
|
|
|
|
for (const line of lines) {
|
|
let ev;
|
|
try { ev = JSON.parse(line); } catch { continue; }
|
|
if (ev.type === 'system' && ev.subtype === 'init') {
|
|
initTools = (ev.tools || []).filter(t => /codegraph/.test(t));
|
|
}
|
|
if (ev.type === 'assistant' && ev.message?.content) {
|
|
for (const block of ev.message.content) {
|
|
if (block.type === 'tool_use') {
|
|
let detail = '';
|
|
if (block.name === 'Task') detail = ` [subagent_type=${block.input?.subagent_type ?? '?'}] ${(block.input?.description ?? '').slice(0,40)}`;
|
|
else if (/codegraph/.test(block.name)) detail = ` ${JSON.stringify(block.input?.query ?? block.input?.task ?? block.input?.symbol ?? '').slice(0,60)}`;
|
|
else if (block.name === 'Bash') detail = ` ${(block.input?.command ?? '').slice(0,50)}`;
|
|
else if (block.name === 'Read') detail = ` ${(block.input?.file_path ?? '').split('/').slice(-1)[0]}`;
|
|
toolCalls.push(`${block.name}${detail}`);
|
|
}
|
|
}
|
|
}
|
|
if (ev.type === 'result') result = ev;
|
|
}
|
|
|
|
console.log(`\n=== ${file.split('/').pop()} ===`);
|
|
console.log(`codegraph tools exposed: ${initTools ? initTools.length : '?'}`);
|
|
console.log(`\nTool calls (${toolCalls.length}):`);
|
|
const counts = {};
|
|
for (const tc of toolCalls) { const n = tc.split(' ')[0]; counts[n] = (counts[n]||0)+1; }
|
|
console.log(' by type:', JSON.stringify(counts));
|
|
toolCalls.forEach((tc, i) => console.log(` ${i+1}. ${tc}`));
|
|
|
|
if (result) {
|
|
const u = result.usage || {};
|
|
const totalIn = (u.input_tokens||0) + (u.cache_read_input_tokens||0) + (u.cache_creation_input_tokens||0);
|
|
console.log(`\nResult: ${result.subtype} | duration ${(result.duration_ms/1000).toFixed(0)}s | turns ${result.num_turns}`);
|
|
console.log(` tokens: in=${totalIn} out=${u.output_tokens||0} | cost $${(result.total_cost_usd||0).toFixed(3)}`);
|
|
}
|