rtk-ai__rtk/scripts/benchmark/run.ts

#!/usr/bin/env bun
/**
 * RTK Full Integration Test Suite — Multipass VM
 *
 * Usage:
 *   bun run scripts/benchmark/run.ts                    # Full suite
 *   bun run scripts/benchmark/run.ts --quick            # Skip slow phases
 *   bun run scripts/benchmark/run.ts --phase 3          # Run specific phase only
 *   bun run scripts/benchmark/run.ts --phase 13         # AI quality tests only
 *   bun run scripts/benchmark/run.ts --llm claude       # Force Claude CLI for quality tests
 *   bun run scripts/benchmark/run.ts --llm gemini       # Force Gemini CLI
 *   bun run scripts/benchmark/run.ts --llm codex        # Force Codex CLI
 *   bun run scripts/benchmark/run.ts --llm ollama       # Force local Ollama
 *   bun run scripts/benchmark/run.ts --llm auto         # Auto-detect (default)
 *
 * Prerequisites:
 *   brew install multipass
 *   At least one LLM CLI for Phase 13: claude, gemini, codex, or ollama
 */

import { $ } from "bun";
import { vmEnsureReady, vmBuildRtk, vmExec, RTK_BIN } from "./lib/vm";
import { testCmd, testSavings, testRewrite, skipTest, getCounts } from "./lib/test";
import { saveReport } from "./lib/report";
import {
  testQuality,
  containsAll,
  containsAny,
  mentionsNumberInRange,
  getQualityResults,
  formatQualityReport,
  setLLMBackend,
  getActiveBackends,
  type QualityTest,
  type LLMBackend,
} from "./lib/quality";

const args = process.argv.slice(2);
const quick = args.includes("--quick");
const phaseArg = args.includes("--phase")
  ? parseInt(args[args.indexOf("--phase") + 1], 10)
  : null;
const phaseOnly = phaseArg !== null && !Number.isNaN(phaseArg) ? phaseArg : null;
if (args.includes("--phase") && phaseOnly === null) {
  console.error("Error: --phase requires a number (e.g. --phase 3)");
  process.exit(1);
}
const llmArg = args.includes("--llm")
  ? args[args.indexOf("--llm") + 1] as LLMBackend
  : "auto";
if (llmArg !== "auto") setLLMBackend(llmArg);

const reportPath = args.includes("--report")
  ? args[args.indexOf("--report") + 1]
  : `${new URL("../../", import.meta.url).pathname.replace(/\/$/, "")}/benchmark-report.txt`;

const PROJECT_ROOT = new URL("../../", import.meta.url).pathname.replace(/\/$/, "");
const RTK = RTK_BIN;

function shouldRun(phase: number): boolean {
  return phaseOnly === null || phaseOnly === phase;
}

function heading(phase: number, title: string) {
  console.log(`\n\x1b[34m[Phase ${phase}] ${title}\x1b[0m`);
}

// ══════════════════════════════════════════════════════════════
// Phase 0: VM Setup
// ══════════════════════════════════════════════════════════════

console.log("\x1b[34m[rtk-test] RTK Full Integration Test Suite\x1b[0m");
console.log(`Project: ${PROJECT_ROOT}`);

await vmEnsureReady();

// ══════════════════════════════════════════════════════════════
// Phase 1: Transfer & Build
// ══════════════════════════════════════════════════════════════

heading(1, "Transfer & Build");
const branch = (await $`git -C ${PROJECT_ROOT} branch --show-current`.text()).trim();
const commit = (await $`git -C ${PROJECT_ROOT} log --oneline -1`.text()).trim();
const buildInfo = await vmBuildRtk(PROJECT_ROOT);

// Binary size check
// ARM Linux release binaries are ~6.5MB (vs ~4MB x86 stripped).
// CLAUDE.md target is <5MB for stripped x86 release builds.
// VM builds are ARM + not fully stripped, so we use a relaxed 8MB limit here.
const sizeLimit = 8_388_608; // 8MB (relaxed for ARM Linux VM)
if (buildInfo.binarySize < sizeLimit) {
  console.log(`  \x1b[32mPASS\x1b[0m | binary size | ${buildInfo.binarySize} bytes < 8MB`);
} else {
  console.log(`  \x1b[31mFAIL\x1b[0m | binary size | ${buildInfo.binarySize} bytes >= 8MB`);
}

// ══════════════════════════════════════════════════════════════
// Phase 2: Cargo Quality (fmt, clippy, test)
// ══════════════════════════════════════════════════════════════

if (shouldRun(2)) {
  heading(2, "Cargo Quality");

  await testCmd(
    "quality:cargo fmt",
    "export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo fmt --all --check 2>&1"
  );

  await testCmd(
    "quality:cargo clippy",
    "export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo clippy --all-targets -- -D warnings 2>&1"
  );

  await testCmd(
    "quality:cargo test",
    "export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo test --all 2>&1"
  );
}

// ══════════════════════════════════════════════════════════════
// Phase 3: Rust Built-in Commands
// ══════════════════════════════════════════════════════════════

if (shouldRun(3)) {
  heading(3, "Rust Built-in Commands");

  // Git
  await testCmd("git:status", `cd /tmp/test-git && ${RTK} git status`);
  await testCmd("git:log", `cd /tmp/test-git && ${RTK} git log -5`);
  await testCmd("git:log --oneline", `cd /tmp/test-git && ${RTK} git log --oneline -10`);
  await testCmd("git:diff", `cd /tmp/test-git && ${RTK} git diff`, "any");
  await testCmd("git:branch", `cd /tmp/test-git && ${RTK} git branch`);
  await testCmd("git:add --dry-run", `cd /tmp/test-git && ${RTK} git add --dry-run .`, "any");

  // Files
  await testCmd("files:ls", `${RTK} ls /home/ubuntu/rtk`);
  await testCmd("files:ls src/", `${RTK} ls /home/ubuntu/rtk/src/`);
  await testCmd("files:ls -R", `${RTK} ls -R /home/ubuntu/rtk/src/`);
  await testCmd("files:read", `${RTK} read /home/ubuntu/rtk/src/main.rs`);
  await testCmd("files:read aggressive", `${RTK} read /home/ubuntu/rtk/src/main.rs -l aggressive`);
  await testCmd("files:smart", `${RTK} smart /home/ubuntu/rtk/src/main.rs`);
  await testCmd("files:find *.rs", `${RTK} find '*.rs' /home/ubuntu/rtk/src/`);
  await testCmd("files:wc", `${RTK} wc /home/ubuntu/rtk/src/main.rs`);
  await testCmd("files:diff", `${RTK} diff /home/ubuntu/rtk/src/main.rs /home/ubuntu/rtk/src/utils.rs`);

  // Search
  await testCmd("search:grep", `${RTK} grep 'fn main' /home/ubuntu/rtk/src/`);

  // Data
  await testCmd("data:json", `${RTK} json /tmp/test-node/package.json`);
  await testCmd("data:deps", `cd /home/ubuntu/rtk && ${RTK} deps`);
  await testCmd("data:env", `${RTK} env`);

  // Runners
  await testCmd("runner:summary", `${RTK} summary 'echo hello world'`);
  // BUG: rtk err swallows exit code — tracked in #846
  await testCmd("runner:err", `${RTK} err false`, "any");
  await testCmd("runner:test", `${RTK} test 'echo ok'`, "any");

  // Logs
  await testCmd("log:large", `${RTK} log /tmp/large.log`);

  // Network
  await testCmd("net:curl", `${RTK} curl https://httpbin.org/get`, "any");

  // GitHub
  await testCmd("gh:pr list", `cd /home/ubuntu/rtk && ${RTK} gh pr list`, "any");

  // Cargo (test project has intentional test failure → exit 101)
  await testCmd("cargo:build", `export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/test-rust && ${RTK} cargo build`);
  await testCmd("cargo:test", `export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/test-rust && ${RTK} cargo test`, 101);
  await testCmd("cargo:clippy", `export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/test-rust && ${RTK} cargo clippy`);

  // Python (test project has intentional failures)
  await testCmd("python:pytest", `cd /tmp/test-python && ${RTK} pytest`, 1);
  await testCmd("python:ruff check", `cd /tmp/test-python && ${RTK} ruff check .`, 1);
  await testCmd("python:mypy", `cd /tmp/test-python && ${RTK} mypy .`, 1);
  await testCmd("python:pip list", `${RTK} pip list`);

  // Go (test project has intentional test failure)
  await testCmd("go:test", `export PATH=$PATH:/usr/local/go/bin && cd /tmp/test-go && ${RTK} go test ./...`, 1);
  await testCmd("go:build", `export PATH=$PATH:/usr/local/go/bin && cd /tmp/test-go && ${RTK} go build .`, 1);
  await testCmd("go:vet", `export PATH=$PATH:/usr/local/go/bin && cd /tmp/test-go && ${RTK} go vet ./...`, 1);
  await testCmd("go:golangci-lint", `export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin && cd /tmp/test-go && ${RTK} golangci-lint run`, 1);

  // TypeScript
  await testCmd("ts:tsc", `cd /tmp/test-node && ${RTK} tsc --noEmit`, "any");

  // Linters
  await testCmd("lint:eslint", `cd /tmp/test-node && ${RTK} lint 'eslint src/'`, "any");
  await testCmd("lint:prettier", `cd /tmp/test-node && ${RTK} prettier --check src/`, "any");

  // Docker
  await testCmd("docker:ps", `${RTK} docker ps`, "any");
  await testCmd("docker:images", `${RTK} docker images`, "any");

  // Kubernetes
  await testCmd("k8s:pods", `${RTK} kubectl pods`, "any");

  // .NET
  await testCmd("dotnet:build", `export DOTNET_ROOT=/usr/local/share/dotnet && export PATH=$PATH:$DOTNET_ROOT && cd /tmp/test-dotnet/TestApp 2>/dev/null && ${RTK} dotnet build || echo 'dotnet skip'`, "any");

  // Meta
  await testCmd("meta:gain", `${RTK} gain`);
  await testCmd("meta:gain --history", `${RTK} gain --history`);
  await testCmd("meta:proxy", `${RTK} proxy echo 'proxy test'`);
  await testCmd("meta:verify", `${RTK} verify`, "any");
}

// ══════════════════════════════════════════════════════════════
// Phase 4: TOML Filter Commands
// ══════════════════════════════════════════════════════════════

if (shouldRun(4)) {
  heading(4, "TOML Filter Commands");

  // System
  await testCmd("toml:df", `${RTK} df -h`);
  await testCmd("toml:du", `${RTK} du -sh /tmp`, "any");
  await testCmd("toml:ps", `${RTK} ps aux`);
  await testCmd("toml:ping", `${RTK} ping -c 2 127.0.0.1`);

  // Build tools
  await testCmd("toml:make", `cd /tmp && ${RTK} make -f Makefile`, "any");
  await testCmd("toml:rsync", `${RTK} rsync --version`);

  // Linters
  await testCmd("toml:shellcheck", `${RTK} shellcheck /tmp/test.sh`, "any");
  await testCmd("toml:hadolint", `${RTK} hadolint /tmp/Dockerfile.bad`, "any");
  await testCmd("toml:yamllint", `${RTK} yamllint /tmp/test.yaml`, "any");
  await testCmd("toml:markdownlint", `${RTK} markdownlint /tmp/test.md`, "any");

  // Cloud/Infra
  await testCmd("toml:terraform", `${RTK} terraform --version`, "any");
  await testCmd("toml:helm", `${RTK} helm version`, "any");
  await testCmd("toml:ansible", `${RTK} ansible-playbook --version`, "any");

  // Mocked tools
  await testCmd("toml:gcloud", `${RTK} gcloud version`);
  await testCmd("toml:shopify", `${RTK} shopify theme check`, "any");
  await testCmd("toml:pio", `${RTK} pio run`, "any");
  await testCmd("toml:quarto", `${RTK} quarto render`, "any");
  await testCmd("toml:sops", `${RTK} sops --version`);
  // Swift ecosystem
  await testCmd("toml:swift build", `${RTK} swift build`, "any");
  await testCmd("toml:swift test", `${RTK} swift test`, "any");
  await testCmd("toml:swift run", `${RTK} swift run`, "any");
  await testCmd("toml:swift package", `${RTK} swift package resolve`, "any");
  await testCmd("toml:swiftlint", `${RTK} swiftlint`, "any");
  await testCmd("toml:swiftformat", `${RTK} swiftformat`, "any");
  await testCmd("toml:kubectl", `${RTK} kubectl version --client`, "any");
}

// ══════════════════════════════════════════════════════════════
// Phase 5: Hook Rewrite Engine
// ══════════════════════════════════════════════════════════════

if (shouldRun(5)) {
  heading(5, "Hook Rewrite Engine");

  // Basic rewrites
  await testRewrite("git status", "rtk git status");
  await testRewrite("git log --oneline -10", "rtk git log --oneline -10");
  await testRewrite("cargo test", "rtk cargo test");
  await testRewrite("cargo build --release", "rtk cargo build --release");
  await testRewrite("docker ps", "rtk docker ps");
  // NOTE: rtk rewrites "kubectl get pods" to "rtk kubectl get pods" (preserves get)
  await testRewrite("kubectl get pods", "rtk kubectl get pods");
  await testRewrite("ruff check", "rtk ruff check");
  await testRewrite("pytest", "rtk pytest");
  await testRewrite("go test", "rtk go test");
  await testRewrite("pnpm list", "rtk pnpm list");
  await testRewrite("gh pr list", "rtk gh pr list");
  await testRewrite("df -h", "rtk df -h");
  await testRewrite("ps aux", "rtk ps aux");

  // Compound
  await testRewrite("cargo test && git status", "rtk cargo test && rtk git status");
  // NOTE: shell strips single quotes in vmExec, so 'msg' becomes msg
  await testRewrite("git add . && git commit -m msg", "rtk git add . && rtk git commit -m msg");

  // No rewrite (shell builtins) — rtk rewrite returns empty string + exit 1
  // We test via testCmd since testRewrite expects non-empty output
  await testCmd("rewrite:cd (no rewrite)", `${RTK} rewrite 'cd /tmp'`, 1);
  await testCmd("rewrite:export (no rewrite)", `${RTK} rewrite 'export FOO=bar'`, 1);
}

// ══════════════════════════════════════════════════════════════
// Phase 6: Exit Code Preservation
// ══════════════════════════════════════════════════════════════

if (shouldRun(6)) {
  heading(6, "Exit Code Preservation");

  // Success
  await testCmd("exit:git status=0", `cd /tmp/test-git && ${RTK} git status`, 0);
  await testCmd("exit:ls=0", `${RTK} ls /tmp`, 0);
  await testCmd("exit:gain=0", `${RTK} gain`, 0);

  // Failures
  // rg returns exit 1 (no match) or 2 (error) — accept both
  await testCmd("exit:grep NOTFOUND", `${RTK} grep NOTFOUND_XYZ_123 /tmp`, "any");
}

// ══════════════════════════════════════════════════════════════
// Phase 7: Token Savings
// ══════════════════════════════════════════════════════════════

if (shouldRun(7)) {
  heading(7, "Token Savings");

  await testSavings(
    "savings:git log",
    "cd /tmp/test-git && git log -20",
    `cd /tmp/test-git && ${RTK} git log -20`,
    60
  );
  await testSavings(
    "savings:ls",
    "ls -la /home/ubuntu/rtk/src/",
    `${RTK} ls /home/ubuntu/rtk/src/`,
    60
  );
  await testSavings(
    "savings:log dedup",
    "cat /tmp/large.log",
    `${RTK} log /tmp/large.log`,
    80
  );
  await testSavings(
    "savings:read aggressive",
    "cat /home/ubuntu/rtk/src/main.rs",
    `${RTK} read /home/ubuntu/rtk/src/main.rs -l aggressive`,
    50
  );
  await testSavings(
    "savings:swift test",
    "swift test",
    `${RTK} swift test`,
    60
  );
  await testSavings(
    "savings:swiftlint",
    "swiftlint",
    `${RTK} swiftlint`,
    20
  );
}

// ══════════════════════════════════════════════════════════════
// Phase 8: Pipe Compatibility
// ══════════════════════════════════════════════════════════════

if (shouldRun(8)) {
  heading(8, "Pipe Compatibility");

  await testCmd("pipe:git status|wc", `cd /tmp/test-git && ${RTK} git status | wc -l`);
  await testCmd("pipe:ls|wc", `${RTK} ls /home/ubuntu/rtk/src/ | wc -l`);
  await testCmd("pipe:grep|head", `${RTK} grep 'fn' /home/ubuntu/rtk/src/ | head -5`);
}

// ══════════════════════════════════════════════════════════════
// Phase 9: Edge Cases
// ══════════════════════════════════════════════════════════════

if (shouldRun(9)) {
  heading(9, "Edge Cases");

  await testCmd("edge:summary true", `${RTK} summary 'true'`, "any");
  await testCmd("edge:grep NOTFOUND", `${RTK} grep NOTFOUND_XYZ /home/ubuntu/rtk/src/`, 1);
  await testCmd("edge:unicode", `echo 'hello world' > /tmp/uni.txt && ${RTK} grep 'hello' /tmp`, "any");
}

// ══════════════════════════════════════════════════════════════
// Phase 10: Performance (skip with --quick)
// ══════════════════════════════════════════════════════════════

if (shouldRun(10) && !quick) {
  heading(10, "Performance");

  // hyperfine
  const { exitCode: hfExist } = await vmExec("command -v hyperfine");
  if (hfExist === 0) {
    const { stdout: hfOut } = await vmExec(
      `cd /tmp/test-git && hyperfine --warmup 3 --min-runs 5 '${RTK} git status' 'git status' --export-json /dev/stdout 2>/dev/null`
    );
    try {
      const hf = JSON.parse(hfOut);
      const rtkMean = (hf.results?.[0]?.mean * 1000).toFixed(1);
      const rawMean = (hf.results?.[1]?.mean * 1000).toFixed(1);
      console.log(`  Startup: rtk=${rtkMean}ms raw=${rawMean}ms`);
    } catch {
      console.log("  hyperfine output parse failed");
    }
  } else {
    skipTest("perf:hyperfine", "not installed");
  }

  // Memory
  const { stdout: memOut } = await vmExec(
    `cd /tmp/test-git && /usr/bin/time -v ${RTK} git status 2>&1 | grep 'Maximum resident'`
  );
  const memKb = parseInt(memOut.match(/(\d+)/)?.[1] ?? "0", 10);
  if (memKb > 0 && memKb < 20000) {
    await testCmd("perf:memory", `echo '${memKb} KB < 20MB'`);
  } else if (memKb > 0) {
    await testCmd("perf:memory", `echo '${memKb} KB >= 20MB' && exit 1`, 0);
  }
} else if (quick && shouldRun(10)) {
  skipTest("perf:hyperfine", "--quick mode");
  skipTest("perf:memory", "--quick mode");
}

// ══════════════════════════════════════════════════════════════
// Phase 11: Concurrency (skip with --quick)
// ══════════════════════════════════════════════════════════════

if (shouldRun(11) && !quick) {
  heading(11, "Concurrency");

  await testCmd(
    "concurrency:10x git status",
    `cd /tmp/test-git && for i in $(seq 1 10); do ${RTK} git status >/dev/null & done; wait`
  );
} else if (quick && shouldRun(11)) {
  skipTest("concurrency:10x", "--quick mode");
}

// ══════════════════════════════════════════════════════════════
// Phase 12: Token Savings on Real Open-Source Projects
// ══════════════════════════════════════════════════════════════

if (shouldRun(12) && !quick) {
  heading(12, "Real Project Token Savings");

  const PATH_PREFIX = "export PATH=$HOME/.cargo/bin:$PATH:/usr/local/go/bin:$HOME/go/bin:$HOME/.local/bin";

  // --- Rust: ripgrep (200+ tests) ---
  await testSavings(
    "real:ripgrep cargo test",
    `${PATH_PREFIX} && cd /tmp/real-ripgrep && cargo test 2>&1`,
    `${PATH_PREFIX} && cd /tmp/real-ripgrep && ${RTK} cargo test 2>&1`,
    90
  );
  await testSavings(
    "real:ripgrep cargo clippy",
    `${PATH_PREFIX} && cd /tmp/real-ripgrep && cargo clippy --all-targets 2>&1`,
    `${PATH_PREFIX} && cd /tmp/real-ripgrep && ${RTK} cargo clippy --all-targets 2>&1`,
    80
  );
  await testSavings(
    "real:ripgrep grep fn",
    `grep -rn 'fn ' /tmp/real-ripgrep/crates/`,
    `${RTK} grep 'fn ' /tmp/real-ripgrep/crates/`,
    80
  );
  await testSavings(
    "real:ripgrep ls -R",
    `ls -laR /tmp/real-ripgrep/crates/`,
    `${RTK} ls -R /tmp/real-ripgrep/crates/`,
    70
  );

  // --- Python: Flask (400+ tests) ---
  await testSavings(
    "real:flask pytest",
    `${PATH_PREFIX} && cd /tmp/real-flask && pytest tests/ 2>&1`,
    `${PATH_PREFIX} && cd /tmp/real-flask && ${RTK} pytest tests/ 2>&1`,
    80
  );
  await testSavings(
    "real:flask grep def",
    `grep -rn 'def ' /tmp/real-flask/src/`,
    `${RTK} grep 'def ' /tmp/real-flask/src/`,
    40
  );
  await testSavings(
    "real:flask ls -R",
    `ls -laR /tmp/real-flask/src/`,
    `${RTK} ls -R /tmp/real-flask/src/`,
    70
  );

  // --- Go: Hugo (1800+ tests) ---
  await testSavings(
    "real:hugo go test",
    `${PATH_PREFIX} && cd /tmp/real-hugo && go test ./hugolib/... 2>&1`,
    `${PATH_PREFIX} && cd /tmp/real-hugo && ${RTK} go test ./hugolib/... 2>&1`,
    60
  );
  await testSavings(
    "real:hugo grep func",
    `grep -rn 'func ' /tmp/real-hugo/hugolib/`,
    `${RTK} grep 'func ' /tmp/real-hugo/hugolib/`,
    70
  );
  await testSavings(
    "real:hugo find *.go",
    `find /tmp/real-hugo -name '*.go' -not -path '*/vendor/*'`,
    `${RTK} find '*.go' /tmp/real-hugo/`,
    70
  );
  await testSavings(
    "real:hugo ls -R",
    `ls -laR /tmp/real-hugo/hugolib/`,
    `${RTK} ls -R /tmp/real-hugo/hugolib/`,
    70
  );

  // --- JS/TS: Vite ---
  await testSavings(
    "real:vite ls -R packages",
    `ls -laR /tmp/real-vite/packages/`,
    `${RTK} ls -R /tmp/real-vite/packages/`,
    70
  );
  await testSavings(
    "real:vite find *.ts",
    `find /tmp/real-vite/packages/vite/src -name '*.ts'`,
    `${RTK} find '*.ts' /tmp/real-vite/packages/vite/src/`,
    60
  );

  // --- C: Redis ---
  await testSavings(
    "real:redis grep struct",
    `grep -rn 'struct ' /tmp/real-redis/src/`,
    `${RTK} grep 'struct ' /tmp/real-redis/src/`,
    70
  );
  await testSavings(
    "real:redis ls -R",
    `ls -laR /tmp/real-redis/src/`,
    `${RTK} ls -R /tmp/real-redis/src/`,
    60
  );

  // --- C#: Newtonsoft.Json ---
  await testSavings(
    "real:newtonsoft find *.cs",
    `find /tmp/real-newtonsoft/Src -name '*.cs'`,
    `${RTK} find '*.cs' /tmp/real-newtonsoft/Src/`,
    80
  );
  await testSavings(
    "real:newtonsoft grep class",
    `grep -rn 'class ' /tmp/real-newtonsoft/Src/`,
    `${RTK} grep 'class ' /tmp/real-newtonsoft/Src/`,
    70
  );
  await testSavings(
    "real:newtonsoft ls -R",
    `ls -laR /tmp/real-newtonsoft/Src/`,
    `${RTK} ls -R /tmp/real-newtonsoft/Src/`,
    60
  );

  // --- Ruby: Sinatra ---
  await testSavings(
    "real:sinatra ls -R",
    `ls -laR /tmp/real-sinatra/lib/`,
    `${RTK} ls -R /tmp/real-sinatra/lib/`,
    60
  );

  // --- Java: Spring Petclinic ---
  await testSavings(
    "real:petclinic find *.java",
    `find /tmp/real-petclinic/src -name '*.java'`,
    `${RTK} find '*.java' /tmp/real-petclinic/src/`,
    40
  );
  await testSavings(
    "real:petclinic ls -R",
    `ls -laR /tmp/real-petclinic/src/`,
    `${RTK} ls -R /tmp/real-petclinic/src/`,
    60
  );

  // --- Cross-project: RTK on itself ---
  await testSavings(
    "real:rtk cargo test (1584)",
    `${PATH_PREFIX} && cd /home/ubuntu/rtk && cargo test --all 2>&1`,
    `${PATH_PREFIX} && cd /home/ubuntu/rtk && ${RTK} cargo test --all 2>&1`,
    95
  );
  await testSavings(
    "real:rtk smart main.rs",
    `cat /home/ubuntu/rtk/src/main.rs`,
    `${RTK} smart /home/ubuntu/rtk/src/main.rs`,
    99
  );
  await testSavings(
    "real:rtk read aggressive",
    `cat /home/ubuntu/rtk/src/main.rs`,
    `${RTK} read /home/ubuntu/rtk/src/main.rs -l aggressive`,
    85
  );

  // --- Logs ---
  await testSavings(
    "real:log dedup 1050 lines",
    `cat /tmp/large.log`,
    `${RTK} log /tmp/large.log`,
    95
  );

  // --- Data ---
  await testSavings(
    "real:pip list",
    `pip list 2>&1`,
    `${RTK} pip list 2>&1`,
    90
  );
  await testSavings(
    "real:ps aux",
    `ps aux`,
    `${RTK} ps aux`,
    80
  );

} else if (quick && shouldRun(12)) {
  skipTest("real:projects", "--quick mode");
}

// ══════════════════════════════════════════════════════════════
// Phase 13: AI Comprehension Quality (skip with --quick)
// ══════════════════════════════════════════════════════════════

if (shouldRun(13) && !quick) {
  heading(13, "AI Comprehension Quality");

  // Quality tests use claude CLI (OAuth auth) or ANTHROPIC_API_KEY as fallback
  {
    const PATH_PREFIX =
      "export PATH=$HOME/.cargo/bin:$PATH:/usr/local/go/bin:$HOME/go/bin:$HOME/.local/bin";

    const qualityTests: QualityTest[] = [
      // --- Cargo test: can the LLM identify failed tests? ---
      {
        name: "cargo test failures",
        rawCmd: `${PATH_PREFIX} && cd /tmp/test-rust && cargo test 2>&1`,
        rtkCmd: `${PATH_PREFIX} && cd /tmp/test-rust && ${RTK} cargo test 2>&1`,
        question:
          "Which tests failed? List the test names and what the error was.",
        extractTruth: (raw) => {
          const failures: string[] = [];
          for (const m of raw.matchAll(/test (\S+) \.\.\. FAILED/g))
            failures.push(m[1]);
          return failures.length > 0 ? failures : ["test_fail"];
        },
        checkAnswer: (answer, truth) => containsAny(answer, truth),
      },

      // --- Cargo test on RTK: can LLM give the count? ---
      {
        name: "cargo test count (RTK 1584)",
        rawCmd: `${PATH_PREFIX} && cd /home/ubuntu/rtk && cargo test --all 2>&1`,
        rtkCmd: `${PATH_PREFIX} && cd /home/ubuntu/rtk && ${RTK} cargo test --all 2>&1`,
        question:
          "How many tests passed? How many failed? Give the exact numbers.",
        extractTruth: () => ["1584", "0 failed"],
        checkAnswer: (answer, _truth) =>
          mentionsNumberInRange(answer, 1580, 1590),
      },

      // --- Cargo clippy: can LLM identify top warnings? ---
      {
        name: "cargo clippy warnings (ripgrep)",
        rawCmd: `${PATH_PREFIX} && cd /tmp/real-ripgrep && cargo clippy --all-targets 2>&1`,
        rtkCmd: `${PATH_PREFIX} && cd /tmp/real-ripgrep && ${RTK} cargo clippy --all-targets 2>&1`,
        question:
          "How many warnings were found? What is the most common warning type?",
        extractTruth: (raw) => {
          const m = raw.match(/(\d+) warnings?/);
          return m ? [m[1]] : ["warning"];
        },
        checkAnswer: (answer, truth) => containsAny(answer, truth),
      },

      // --- pytest Flask: can LLM identify test results? ---
      {
        name: "pytest results (Flask)",
        rawCmd: `${PATH_PREFIX} && cd /tmp/real-flask && pytest tests/ -x --tb=short 2>&1 | head -100`,
        rtkCmd: `${PATH_PREFIX} && cd /tmp/real-flask && ${RTK} pytest tests/ -x --tb=short 2>&1`,
        question:
          "Did all tests pass or were there failures? How many tests were run?",
        extractTruth: (raw) => {
          const m = raw.match(/(\d+) passed/);
          return m ? [m[1], "passed"] : ["passed"];
        },
        checkAnswer: (answer, truth) => containsAny(answer, truth),
      },

      // --- Go test Hugo: can LLM identify failures? ---
      {
        name: "go test results (Hugo)",
        rawCmd: `${PATH_PREFIX} && cd /tmp/real-hugo && go test ./hugolib/... 2>&1 | head -200`,
        rtkCmd: `${PATH_PREFIX} && cd /tmp/real-hugo && ${RTK} go test ./hugolib/... 2>&1`,
        question:
          "Did the tests pass or fail? How many packages were tested?",
        extractTruth: (raw) => {
          const passed = raw.includes("ok") ? ["pass", "ok"] : ["fail"];
          return passed;
        },
        checkAnswer: (answer, truth) => containsAny(answer, truth),
      },

      // --- grep: can LLM identify files with matches? ---
      {
        name: "grep results (ripgrep fn)",
        rawCmd: `grep -rn 'fn main' /tmp/real-ripgrep/crates/`,
        rtkCmd: `${RTK} grep 'fn main' /tmp/real-ripgrep/crates/`,
        question:
          "Which files contain 'fn main'? List the file paths.",
        extractTruth: (raw) => {
          const files = new Set<string>();
          for (const line of raw.split("\n")) {
            const m = line.match(/^([^:]+):/);
            if (m) files.add(m[1].split("/").pop() ?? "");
          }
          return [...files].slice(0, 3);
        },
        checkAnswer: (answer, truth) => containsAny(answer, truth),
      },

      // --- ls -R: can LLM identify the structure? ---
      {
        name: "ls -R structure (Redis)",
        rawCmd: `ls -laR /tmp/real-redis/src/`,
        rtkCmd: `${RTK} ls -R /tmp/real-redis/src/`,
        question:
          "How many .c files are there approximately? What are some of the file names?",
        extractTruth: (raw) => {
          const cFiles = raw.split("\n").filter((l) => l.includes(".c")).length;
          return [String(cFiles), "server", "redis"];
        },
        checkAnswer: (answer, truth) => containsAny(answer, truth),
      },

      // --- read aggressive: can LLM understand the code? ---
      {
        name: "read aggressive (RTK main.rs)",
        rawCmd: `cat /home/ubuntu/rtk/src/main.rs | head -200`,
        rtkCmd: `${RTK} read /home/ubuntu/rtk/src/main.rs -l aggressive`,
        question:
          "What is this program? What language is it in? Name 3 main features or modules.",
        extractTruth: () => ["rust", "cli", "rtk"],
        checkAnswer: (answer, truth) => containsAny(answer, truth),
      },

      // --- smart: can LLM understand from 20 tokens? ---
      {
        name: "smart summary (RTK main.rs)",
        rawCmd: `cat /home/ubuntu/rtk/src/main.rs | head -200`,
        rtkCmd: `${RTK} smart /home/ubuntu/rtk/src/main.rs`,
        question: "What kind of program is this? What does it do?",
        extractTruth: () => ["cli", "command"],
        checkAnswer: (answer, truth) => containsAny(answer, truth),
      },

      // --- log dedup: can LLM identify error types? ---
      {
        name: "log dedup (1050 lines)",
        rawCmd: `cat /tmp/large.log`,
        rtkCmd: `${RTK} log /tmp/large.log`,
        question:
          "What types of log messages are present (INFO, WARN, ERROR, FATAL)? What is the most serious issue?",
        extractTruth: () => ["fatal", "out of memory"],
        checkAnswer: (answer, truth) => containsAll(answer, truth),
      },

      // --- pip list: can LLM identify installed packages? ---
      {
        name: "pip list packages",
        rawCmd: `pip list 2>&1`,
        rtkCmd: `${RTK} pip list 2>&1`,
        question: "How many Python packages are installed? Name 3 of them.",
        extractTruth: (raw) => {
          const lines = raw.trim().split("\n").filter((l) => !l.startsWith("-"));
          return [String(lines.length - 1)];
        },
        checkAnswer: (answer, _truth) =>
          mentionsNumberInRange(answer, 10, 500),
      },

      // --- ps aux: can LLM find key processes? ---
      {
        name: "ps aux processes",
        rawCmd: `ps aux`,
        rtkCmd: `${RTK} ps aux`,
        question:
          "What are the main processes running? Is there a database or web server?",
        extractTruth: () => ["root", "sbin"],
        checkAnswer: (answer, _truth) =>
          answer.toLowerCase().includes("process") ||
          answer.toLowerCase().includes("running"),
      },
    ];

    for (const test of qualityTests) {
      await testQuality(test, vmExec);
    }

    // Print quality summary
    const q = getQualityResults();
    console.log(
      `\n  Quality: ${q.qualityPreserved}/${q.totalTests} preserved (${q.avgSavings}% avg savings)`
    );
    if (q.qualityLost > 0) {
      console.log(
        `  \x1b[31m${q.qualityLost} tests lost quality — RTK removed critical info\x1b[0m`
      );
    }
  }
} else if (quick && shouldRun(13)) {
  skipTest("quality:all", "--quick mode");
}

// ══════════════════════════════════════════════════════════════
// Report
// ══════════════════════════════════════════════════════════════

const report = await saveReport(
  { ...buildInfo, branch, commit },
  reportPath
);

console.log("\n" + report);

// Append quality report if tests were run
const qr = getQualityResults();
if (qr.totalTests > 0) {
  const qualityMd = formatQualityReport();
  console.log("\n" + qualityMd);
  // Append to report file
  await Bun.write(reportPath.replace(".txt", "-quality.md"), qualityMd);
}

const { total, passed, failed, skipped } = getCounts();
const passRate = total > 0 ? Math.round((passed * 100) / total) : 0;

if (failed === 0) {
  console.log(`\n\x1b[32m  READY FOR RELEASE — ${passed}/${total} (${passRate}%)\x1b[0m\n`);
  process.exit(0);
} else {
  console.log(`\n\x1b[31m  NOT READY — ${failed} failures — ${passed}/${total} (${passRate}%)\x1b[0m\n`);
  process.exit(1);
}