vercel__eve/apps/benchmarks/scripts/analyze.mjs

#!/usr/bin/env node

// Reads a benchmark results directory and reports, per case, the signals that
// matter when tuning the generated AGENTS.md and the shipped docs: outcome,
// agent cost, which docs pages the agent opened, and which grader assertions
// failed. Docs reads are recovered from shell commands because agents read
// `node_modules/eve/docs` with `cat`/`sed`/`grep` rather than a read tool.

import { existsSync, readdirSync, readFileSync } from "node:fs";
import { basename, join, resolve } from "node:path";
import { parseArgs } from "node:util";

const { values, positionals } = parseArgs({
  args: process.argv.slice(2),
  allowPositionals: true,
  options: { json: { type: "boolean" }, docs: { type: "boolean" }, help: { type: "boolean" } },
  strict: true,
});

if (values.help) {
  console.log(`Usage:
  node scripts/analyze.mjs [results-dir] [--docs] [--json]

  results-dir  A run directory, a case directory, a timestamp directory, or a
               treatment directory. Defaults to the newest timestamp directory.
  --docs       Aggregate docs-page reads across cases instead of per-case detail.
  --json       Emit the collected records as JSON.`);
  process.exit(0);
}

const resultsRoot = resolve(import.meta.dirname, "../results");
const target =
  positionals[0] === undefined ? newestTimestamp(resultsRoot) : resolve(positionals[0]);
const runs = collectRuns(target);
if (runs.length === 0) throw new Error(`No benchmark runs found under ${target}.`);

const records = runs.map((run) => analyzeRun(run));
if (values.json) console.log(JSON.stringify(records, null, 2));
else if (values.docs) reportDocs(records);
else reportRuns(records, target);

function newestTimestamp(root) {
  const treatments = directories(root);
  const stamps = treatments.flatMap((treatment) =>
    directories(treatment).map((path) => ({ path, name: basename(path) })),
  );
  if (stamps.length === 0) throw new Error(`No results under ${root}.`);
  stamps.sort((a, b) => a.name.localeCompare(b.name));
  return stamps.at(-1).path;
}

function directories(root) {
  if (!existsSync(root)) return [];
  return readdirSync(root, { withFileTypes: true })
    .filter((entry) => entry.isDirectory())
    .map((entry) => join(root, entry.name));
}

// Accepts any depth: run dir, case dir, timestamp dir, or treatment dir.
function collectRuns(root) {
  if (existsSync(join(root, "result.json"))) return [root];
  return directories(root)
    .flatMap((child) => collectRuns(child))
    .sort();
}

function analyzeRun(runPath) {
  const result = readJson(join(runPath, "result.json"));
  const raw = readRawTranscript(join(runPath, "transcript-raw.jsonl"));
  const timings = readTimings(runPath);
  const toolCalls = raw.flatMap((entry) => entry.toolCalls);
  const commands = toolCalls.flatMap((call) => (call.command === undefined ? [] : [call.command]));
  const usage = raw.reduce(
    (total, entry) => ({
      inputTokens: total.inputTokens + (entry.usage?.inputTokens ?? 0),
      outputTokens: total.outputTokens + (entry.usage?.outputTokens ?? 0),
      reasoningTokens: total.reasoningTokens + (entry.usage?.reasoningTokens ?? 0),
      cachedInputTokens: total.cachedInputTokens + (entry.usage?.cachedInputTokens ?? 0),
    }),
    { inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cachedInputTokens: 0 },
  );
  return {
    case: basename(resolve(runPath, "..")),
    run: basename(runPath),
    path: runPath,
    status: result.status,
    durationSeconds: round(result.duration),
    agentSeconds: round(sumPhases(timings, /^agent\.turn\.\d+$/) / 1000),
    setupSeconds: round(sumPhases(timings, /^(subject|session)\./) / 1000),
    turns: raw.filter((entry) => entry.role === "user").length,
    toolCalls: toolCalls.length,
    commands: commands.length,
    usage,
    docsRead: docsCounts(docsSequence(toolCalls)),
    // Which page the agent opened first. `README.md` means the routing index did
    // its job; anything else means the agent guessed or searched its way in.
    docsEntry: docsSequence(toolCalls)[0] ?? "(none)",
    registryCalls: commands.filter((command) => /\beve\s+(registry|add)\b/.test(command)).length,
    failedCommands: failedCommands(commands),
    stalls: timings
      .filter(
        (entry) => /^agent\.turn\.\d+\.summary$/.test(entry.phase) && entry.outcome !== "success",
      )
      .map((entry) => `${entry.phase}: ${entry.details?.stall ?? "did not finish"}`),
    failures: graderFailures(join(runPath, "outputs/eval.txt")),
    scriptFailures: scriptFailures(runPath),
  };
}

// Recovers `docs/<page>` paths from any tool input that mentions them, so a
// `cat`, `sed -n`, `grep -r`, or read-tool path all count as one read each. Call
// order is preserved: the first page tells you which entry point the agent used.
function docsSequence(toolCalls) {
  const pages = [];
  for (const call of toolCalls) {
    for (const text of [call.command ?? "", ...call.paths]) {
      for (const match of text.matchAll(/eve\/docs\/([\w./-]*\.mdx?)/g)) pages.push(match[1]);
      // A bare directory listing or a glob read counts as a directory probe.
      for (const match of text.matchAll(/eve\/docs\/([\w/-]+)\/(?:\*|$|\s)/g)) {
        pages.push(`${match[1]}/*`);
      }
    }
  }
  return pages;
}

function docsCounts(pages) {
  const counts = new Map();
  for (const page of pages) counts.set(page, (counts.get(page) ?? 0) + 1);
  return Object.fromEntries([...counts].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])));
}

function failedCommands(commands) {
  // Cheap proxy for wasted effort: repeated identical commands and probes into
  // paths that the scaffold does not have.
  const seen = new Map();
  for (const command of commands) seen.set(command, (seen.get(command) ?? 0) + 1);
  return [...seen].filter(([, count]) => count > 1).map(([command, count]) => ({ command, count }));
}

function graderFailures(path) {
  if (!existsSync(path)) return [];
  const output = readFileSync(path, "utf8");
  const failures = [...output.matchAll(/^\s*(?:×|✕|FAIL)\s+(.+)$/gm)].map((match) =>
    match[1].trim(),
  );
  return [...new Set(failures)];
}

function scriptFailures(runPath) {
  const scripts = join(runPath, "outputs/scripts");
  if (!existsSync(scripts)) return [];
  return readdirSync(scripts)
    .filter((name) =>
      /error|not found|exit code [1-9]/i.test(readFileSync(join(scripts, name), "utf8")),
    )
    .map((name) => basename(name, ".txt"));
}

function readRawTranscript(path) {
  if (!existsSync(path)) return [];
  return readFileSync(path, "utf8")
    .trim()
    .split("\n")
    .filter((line) => line.length > 0)
    .map((line) => {
      const entry = JSON.parse(line);
      const content = entry.message?.content;
      const blocks = Array.isArray(content) ? content : [];
      return {
        role: entry.type,
        usage: entry.message?.usage,
        toolCalls: blocks
          .filter((block) => block.type === "tool_use")
          .map((block) => ({
            name: block.name,
            command: typeof block.input?.command === "string" ? block.input.command : undefined,
            paths: [block.input?.filePath, block.input?.path, block.input?.pattern].filter(
              (value) => typeof value === "string",
            ),
          })),
      };
    });
}

function readTimings(runPath) {
  const path = join(runPath, "project/benchmark/timings.json");
  return existsSync(path) ? readJson(path) : [];
}

function sumPhases(timings, pattern) {
  return timings
    .filter((entry) => pattern.test(entry.phase))
    .reduce((total, entry) => total + entry.durationMs, 0);
}

function readJson(path) {
  return JSON.parse(readFileSync(path, "utf8"));
}

function round(value) {
  return typeof value === "number" ? Math.round(value * 10) / 10 : 0;
}

function reportRuns(records, target) {
  console.log(`# ${target}\n`);
  for (const record of records) {
    const mark = record.status === "passed" ? "PASS" : "FAIL";
    console.log(
      `${mark}  ${record.case} ${record.run}  ${record.durationSeconds}s total / ${record.agentSeconds}s agent / ${record.setupSeconds}s setup`,
    );
    console.log(
      `      turns=${record.turns} tools=${record.toolCalls} in=${record.usage.inputTokens} out=${record.usage.outputTokens} reasoning=${record.usage.reasoningTokens} registry=${record.registryCalls}`,
    );
    const docs = Object.entries(record.docsRead);
    console.log(
      `      docs: entered at ${record.docsEntry}${docs.length === 0 ? "" : ` — ${docs.map(([page, count]) => (count > 1 ? `${page}×${count}` : page)).join(", ")}`}`,
    );
    for (const stall of record.stalls) console.log(`      ! ${stall}`);
    for (const failure of record.failures) console.log(`      ! ${failure}`);
    for (const script of record.scriptFailures) console.log(`      ! script ${script}`);
    const repeats = record.failedCommands.filter((entry) => entry.count > 2);
    for (const repeat of repeats.slice(0, 3)) {
      console.log(`      ~ repeated ×${repeat.count}: ${repeat.command.slice(0, 100)}`);
    }
    console.log("");
  }
  const passed = records.filter((record) => record.status === "passed").length;
  console.log(`${passed}/${records.length} passed`);
}

function reportDocs(records) {
  const totals = new Map();
  for (const record of records) {
    for (const [page, count] of Object.entries(record.docsRead)) {
      const entry = totals.get(page) ?? { reads: 0, cases: new Set() };
      entry.reads += count;
      entry.cases.add(record.case);
      totals.set(page, entry);
    }
  }
  const rows = [...totals].sort((a, b) => b[1].reads - a[1].reads || a[0].localeCompare(b[0]));
  console.log("reads  cases  page");
  for (const [page, entry] of rows) {
    console.log(
      `${String(entry.reads).padStart(5)}  ${String(entry.cases.size).padStart(5)}  ${page}`,
    );
    console.log(`                 ${[...entry.cases].sort().join(", ")}`);
  }
}