mirror of
https://github.com/kunchenguid/axi.git
synced 2026-09-14 20:26:53 +08:00
1609 lines
61 KiB
HTML
1609 lines
61 KiB
HTML
<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<title>AXI: Agent eXperience Interface</title>
|
|
<link
|
|
rel="stylesheet"
|
|
href="https://kunchenguid.github.io/kunchenguid-design-system/css/design-system.css"
|
|
/>
|
|
<script
|
|
src="https://kunchenguid.github.io/kunchenguid-design-system/js/design-system.js"
|
|
defer
|
|
></script>
|
|
<style>
|
|
.cell-fail {
|
|
color: #b91c1c;
|
|
}
|
|
.row-category td {
|
|
font-style: italic;
|
|
color: var(--color-text-muted);
|
|
}
|
|
.chart-rows {
|
|
display: flex;
|
|
flex-direction: column;
|
|
gap: 0.375rem;
|
|
}
|
|
.chart-row {
|
|
display: flex;
|
|
align-items: center;
|
|
gap: 0.5rem;
|
|
}
|
|
.chart-label {
|
|
flex-shrink: 0;
|
|
width: 7rem;
|
|
text-align: right;
|
|
font-family: var(--font-mono);
|
|
font-size: var(--text-xs);
|
|
color: var(--color-text-muted);
|
|
}
|
|
.chart-bar-track {
|
|
flex: 1;
|
|
height: 1.25rem;
|
|
background: var(--color-border);
|
|
}
|
|
.chart-bar {
|
|
height: 100%;
|
|
background: var(--color-text-muted);
|
|
}
|
|
.chart-bar.bar-highlight {
|
|
background: var(--color-accent);
|
|
}
|
|
.chart-value {
|
|
flex-shrink: 0;
|
|
width: 3.5rem;
|
|
font-family: var(--font-mono);
|
|
font-size: var(--text-xs);
|
|
color: var(--color-text-muted);
|
|
}
|
|
.chart-value.value-highlight {
|
|
color: var(--color-accent);
|
|
font-weight: 600;
|
|
}
|
|
.copyable {
|
|
position: relative;
|
|
}
|
|
.btn-copy {
|
|
position: absolute;
|
|
top: 0.375rem;
|
|
right: 0.375rem;
|
|
background: none;
|
|
border: 1px solid var(--color-border);
|
|
border-radius: var(--border-radius);
|
|
padding: 0.25rem;
|
|
cursor: pointer;
|
|
color: var(--color-text-muted);
|
|
line-height: 0;
|
|
}
|
|
.btn-copy:hover {
|
|
color: var(--color-text);
|
|
}
|
|
.btn-copy svg {
|
|
width: 1rem;
|
|
height: 1rem;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<!-- Header -->
|
|
<header class="section" style="border-top: none">
|
|
<p class="section-label">AXI — Agent eXperience Interface</p>
|
|
<h1 data-annotate>Let's build apps that [agents] love.</h1>
|
|
<p class="text-lg">
|
|
10 design principles for agent-ergonomic CLI. 100% task success at the
|
|
lowest cost — beating both MCP and raw CLI across 425 benchmark
|
|
runs.
|
|
</p>
|
|
|
|
<div class="grid grid--2 mt-xl">
|
|
<div>
|
|
<p class="section-label">Success Rate</p>
|
|
<div class="chart-rows">
|
|
<div class="chart-row">
|
|
<span class="chart-label">gh-cli</span>
|
|
<div class="chart-bar-track">
|
|
<div class="chart-bar" style="width: 86%"></div>
|
|
</div>
|
|
<span class="chart-value">86%</span>
|
|
</div>
|
|
<div class="chart-row">
|
|
<span class="chart-label">gh-mcp</span>
|
|
<div class="chart-bar-track">
|
|
<div class="chart-bar" style="width: 87%"></div>
|
|
</div>
|
|
<span class="chart-value">87%</span>
|
|
</div>
|
|
<div class="chart-row">
|
|
<span class="chart-label">gh-mcp-search</span>
|
|
<div class="chart-bar-track">
|
|
<div class="chart-bar" style="width: 82%"></div>
|
|
</div>
|
|
<span class="chart-value">82%</span>
|
|
</div>
|
|
<div class="chart-row">
|
|
<span class="chart-label">gh-mcp-code</span>
|
|
<div class="chart-bar-track">
|
|
<div class="chart-bar" style="width: 84%"></div>
|
|
</div>
|
|
<span class="chart-value">84%</span>
|
|
</div>
|
|
<div class="chart-row">
|
|
<span class="chart-label">gh-axi</span>
|
|
<div class="chart-bar-track">
|
|
<div
|
|
class="chart-bar bar-highlight"
|
|
style="width: 100%"
|
|
></div>
|
|
</div>
|
|
<span class="chart-value value-highlight">100%</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div>
|
|
<p class="section-label">Avg Cost per Task</p>
|
|
<div class="chart-rows">
|
|
<div class="chart-row">
|
|
<span class="chart-label">gh-cli</span>
|
|
<div class="chart-bar-track">
|
|
<div class="chart-bar" style="width: 36%"></div>
|
|
</div>
|
|
<span class="chart-value">$0.054</span>
|
|
</div>
|
|
<div class="chart-row">
|
|
<span class="chart-label">gh-mcp</span>
|
|
<div class="chart-bar-track">
|
|
<div class="chart-bar" style="width: 100%"></div>
|
|
</div>
|
|
<span class="chart-value">$0.148</span>
|
|
</div>
|
|
<div class="chart-row">
|
|
<span class="chart-label">gh-mcp-search</span>
|
|
<div class="chart-bar-track">
|
|
<div class="chart-bar" style="width: 99%"></div>
|
|
</div>
|
|
<span class="chart-value">$0.147</span>
|
|
</div>
|
|
<div class="chart-row">
|
|
<span class="chart-label">gh-mcp-code</span>
|
|
<div class="chart-bar-track">
|
|
<div class="chart-bar" style="width: 68%"></div>
|
|
</div>
|
|
<span class="chart-value">$0.101</span>
|
|
</div>
|
|
<div class="chart-row">
|
|
<span class="chart-label">gh-axi</span>
|
|
<div class="chart-bar-track">
|
|
<div class="chart-bar bar-highlight" style="width: 34%"></div>
|
|
</div>
|
|
<span class="chart-value value-highlight">$0.050</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="grid grid--2 mt-xl">
|
|
<div class="card">
|
|
<p class="section-label">Try it now</p>
|
|
<p class="text-sm text-muted">
|
|
<code>gh-axi</code> is a wrapper around the <code>gh</code> CLI,
|
|
optimized for agent ergonomics.
|
|
</p>
|
|
<pre
|
|
class="copyable"
|
|
><code>npm install -g gh-axi</code><button class="btn-copy" onclick="copyBlock(this)" aria-label="Copy to clipboard"><svg class="icon-copy" fill="none" stroke="currentColor" stroke-width="1.5" viewBox="0 0 24 24"><rect x="9" y="9" width="13" height="13" rx="2" ry="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg><svg class="icon-check" fill="none" stroke="currentColor" stroke-width="2" viewBox="0 0 24 24" style="display:none"><path stroke-linecap="round" stroke-linejoin="round" d="M4.5 12.75l6 6 9-13.5"/></svg></button></pre>
|
|
<p class="text-sm text-muted mb-0">
|
|
Add to your <code>CLAUDE.md</code> or <code>AGENTS.md</code>:
|
|
</p>
|
|
<pre
|
|
class="copyable"
|
|
><code>Use `gh-axi` (replacement for `gh` CLI) for all GitHub operations.</code><button class="btn-copy" onclick="copyBlock(this)" aria-label="Copy to clipboard"><svg class="icon-copy" fill="none" stroke="currentColor" stroke-width="1.5" viewBox="0 0 24 24"><rect x="9" y="9" width="13" height="13" rx="2" ry="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg><svg class="icon-check" fill="none" stroke="currentColor" stroke-width="2" viewBox="0 0 24 24" style="display:none"><path stroke-linecap="round" stroke-linejoin="round" d="M4.5 12.75l6 6 9-13.5"/></svg></button></pre>
|
|
</div>
|
|
<div class="card">
|
|
<p class="section-label">Build your own AXI</p>
|
|
<p class="text-sm text-muted">
|
|
Install the AXI skill to get the design guidelines and
|
|
scaffolding:
|
|
</p>
|
|
<pre
|
|
class="copyable"
|
|
><code>npx skills add kunchenguid/axi</code><button class="btn-copy" onclick="copyBlock(this)" aria-label="Copy to clipboard"><svg class="icon-copy" fill="none" stroke="currentColor" stroke-width="1.5" viewBox="0 0 24 24"><rect x="9" y="9" width="13" height="13" rx="2" ry="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg><svg class="icon-check" fill="none" stroke="currentColor" stroke-width="2" viewBox="0 0 24 24" style="display:none"><path stroke-linecap="round" stroke-linejoin="round" d="M4.5 12.75l6 6 9-13.5"/></svg></button></pre>
|
|
</div>
|
|
</div>
|
|
|
|
<p class="text-sm text-muted mt-md">
|
|
<a href="https://github.com/kunchenguid/axi" target="_blank"
|
|
>Github Repo</a
|
|
>
|
|
· Kun Chen Presents
|
|
</p>
|
|
</header>
|
|
|
|
<!-- 1. Introduction -->
|
|
<section class="section">
|
|
<p class="section-label">Premise</p>
|
|
<h2 data-annotate>Neither CLI nor MCP gives [agents] enough love.</h2>
|
|
<p>
|
|
AI agents interact with external services through two dominant
|
|
paradigms. The first is <strong>shell-based CLI execution</strong>:
|
|
the agent runs commands like <code>gh issue list</code> and parses the
|
|
text output. The second is
|
|
<strong>structured tool protocols</strong> like MCP (Model Context
|
|
Protocol), where the agent invokes typed tool functions through the
|
|
hosting framework’s native tool-calling interface.
|
|
</p>
|
|
<p>Both approaches have significant problems for agents:</p>
|
|
<ul>
|
|
<li>
|
|
<strong>CLI: verbose, metadata-sparse output.</strong>
|
|
Human-oriented CLIs produce tab-separated tables, ANSI formatting,
|
|
and per-item descriptions that waste token budget. Worse, they omit
|
|
critical metadata — a label list returns 30 items with no
|
|
total count, and the agent reports 30 when the actual answer is 126.
|
|
</li>
|
|
<li>
|
|
<strong>MCP: massive schema overhead.</strong> MCP tool definitions
|
|
consume 137K–176K input tokens per task in our benchmark,
|
|
compared to 46–47K for CLI conditions. This 3–4×
|
|
context inflation translates directly to 2–3× higher
|
|
cost per task ($0.10–0.15 vs. $0.05).
|
|
</li>
|
|
<li>
|
|
<strong>Both: poor discoverability.</strong> CLI agents must guess
|
|
subcommands or read <code>--help</code>. MCP agents with lazy
|
|
loading waste turns on tool discovery. Neither provides in-context
|
|
guidance on what to do next.
|
|
</li>
|
|
</ul>
|
|
<p>
|
|
Recent debate has framed this as “MCP vs. CLI”
|
|
<a href="#ref-smithery">[2]</a>, but the real question is neither
|
|
<em>which</em> protocol nor <em>which</em> transport, but rather
|
|
<em>what design principles</em> make an agent-tool interface
|
|
effective.
|
|
</p>
|
|
<p>
|
|
AXI is
|
|
<strong>10 principles for agent-ergonomic CLI design</strong> that
|
|
treat token budget as a first-class constraint. AXI achieves the
|
|
reliability advantages MCP promises (structured output,
|
|
discoverability) at the cost profile of a CLI.
|
|
</p>
|
|
|
|
<hr class="divider" />
|
|
|
|
<h3>Contributions</h3>
|
|
<ol>
|
|
<li>
|
|
Ten design principles for agent-ergonomic CLI tools, organized into
|
|
four categories.
|
|
</li>
|
|
<li>
|
|
A reference implementation (<code>gh-axi</code>) demonstrating the
|
|
principles on the GitHub platform.
|
|
</li>
|
|
<li>
|
|
A 425-run benchmark comparing five agent-tool interface conditions,
|
|
with trajectory-level analysis showing where and why each interface
|
|
fails.
|
|
</li>
|
|
</ol>
|
|
</section>
|
|
|
|
<!-- 2. Related Work -->
|
|
<section class="section">
|
|
<p class="section-label">Context</p>
|
|
<h2>Related Work</h2>
|
|
|
|
<h4>Model Context Protocol (MCP)</h4>
|
|
<p>
|
|
Anthropic’s Model Context Protocol
|
|
<a href="#ref-mcp">[1]</a> provides a standardized way to connect AI
|
|
models to external tools through structured, typed tool definitions.
|
|
While MCP offers type-safe schemas and discoverable tool catalogs, the
|
|
schema overhead is substantial. The evaluation below quantifies this
|
|
cost: MCP tool definitions consume 3–4× more context
|
|
tokens than equivalent CLI conditions, translating to 2–3×
|
|
higher cost.
|
|
</p>
|
|
|
|
<h4>MCP vs. CLI benchmarks</h4>
|
|
<p>
|
|
Mao and Pradhan <a href="#ref-smithery">[2]</a> benchmarked 756 runs
|
|
comparing raw APIs, CLI, and native MCP across GitHub, Linear, and
|
|
Singapore Bus APIs. They found native MCP achieved 91.7% success vs.
|
|
83.3% for CLI, with CLI using 2.9× more billed tokens. However,
|
|
their CLI was auto-generated from API specs, not hand-designed for
|
|
agents. They note: “What this does not settle is whether a
|
|
hand-crafted, agent-first CLI could close the remaining gap.”
|
|
AXI answers this question directly — an agent-first CLI not only
|
|
closes the gap but surpasses MCP on every metric.
|
|
</p>
|
|
|
|
<h4>Code execution with MCP</h4>
|
|
<p>
|
|
Jones and Kelly <a href="#ref-anthropic-code">[3]</a> identified two
|
|
scaling problems with direct MCP tool calls: tool definitions overload
|
|
the context window, and intermediate results consume additional tokens
|
|
as they pass through the model between calls. They proposed presenting
|
|
MCP tools as a code API that the agent programs against, so that data
|
|
flows through the execution environment rather than the context
|
|
window. Varda and Pai
|
|
<a href="#ref-cloudflare-code">[4]</a> independently arrived at the
|
|
same insight, coining the term “Code Mode” and reporting
|
|
that LLMs handle complex, multi-step tool interactions better when
|
|
writing TypeScript than when making direct tool calls. The
|
|
<code>mcp-with-code-mode</code> condition in this benchmark evaluates
|
|
this approach: it is the cheapest MCP condition ($0.101/task vs.
|
|
$0.147–0.148), confirming that code execution reduces MCP
|
|
overhead, but it remains 2× more expensive than AXI and
|
|
introduces new failure modes from runtime errors in generated code.
|
|
</p>
|
|
</section>
|
|
|
|
<!-- 3. The 10 Principles -->
|
|
<section class="section">
|
|
<p class="section-label">The Model</p>
|
|
<h2 data-annotate>The [10 principles].</h2>
|
|
|
|
<div class="card mt-xl">
|
|
<ol>
|
|
<li>
|
|
<strong>Token-efficient output</strong> — Use TOON format
|
|
for ~40% token savings over JSON
|
|
</li>
|
|
<li>
|
|
<strong>Minimal default schemas</strong> — 3–4 fields
|
|
per list item, not 10+
|
|
</li>
|
|
<li>
|
|
<strong>Content truncation</strong> — Truncate large text
|
|
fields with size hints and escape hatches
|
|
</li>
|
|
<li>
|
|
<strong>Content first</strong> — Prefer showing actual data,
|
|
not a wall of help text
|
|
</li>
|
|
<li>
|
|
<strong>Contextual disclosure</strong> — Append relevant
|
|
next-step commands after output, not all upfront
|
|
</li>
|
|
<li>
|
|
<strong>Consistent way to get help</strong> — Concise
|
|
per-subcommand reference for when agents need it
|
|
</li>
|
|
<li>
|
|
<strong>Pre-computed fields</strong> — Include aggregated
|
|
statuses that eliminate round trips
|
|
</li>
|
|
<li>
|
|
<strong>Definitive empty states</strong> — Explicit “0
|
|
results” rather than ambiguous empty output
|
|
</li>
|
|
<li>
|
|
<strong>Graceful error handling</strong> — Idempotent
|
|
mutations, structured errors, no interactive prompts
|
|
</li>
|
|
<li>
|
|
<strong>Output discipline</strong> — stdout for data, stderr
|
|
for debug; clean exit codes
|
|
</li>
|
|
</ol>
|
|
</div>
|
|
|
|
<!-- Output Efficiency -->
|
|
<h3 class="mt-xl">Output Efficiency</h3>
|
|
|
|
<h4><span class="text-accent">1.</span> Token-efficient output</h4>
|
|
<p>
|
|
Use TOON (Token-Optimized Object Notation) format instead of JSON or
|
|
tab-separated tables. TOON omits braces, quotes, and commas, yielding
|
|
approximately 40% token savings over equivalent JSON while remaining
|
|
unambiguous to LLMs.
|
|
</p>
|
|
|
|
<div class="grid grid--2">
|
|
<div class="card">
|
|
<p class="section-label">Conventional (JSON)</p>
|
|
<pre><code>[{"number":42,"title":"Fix login bug","state":"open",
|
|
"author":"alice","labels":["bug","P1"]},
|
|
{"number":43,"title":"Add dark mode","state":"open",
|
|
"author":"bob","labels":["feature"]}]</code></pre>
|
|
</div>
|
|
<div class="card">
|
|
<p class="section-label">AXI (TOON)</p>
|
|
<pre><code>issues[2]{number,title,state}:
|
|
42,Fix login bug,open
|
|
43,Add dark mode,open</code></pre>
|
|
</div>
|
|
</div>
|
|
|
|
<h4><span class="text-accent">2.</span> Minimal default schemas</h4>
|
|
<p>
|
|
Return 3–4 fields per list item by default, not 10+. Agents
|
|
rarely need all available fields and can request additional ones
|
|
explicitly via a <code>--fields</code> flag.
|
|
</p>
|
|
|
|
<h4><span class="text-accent">3.</span> Content truncation</h4>
|
|
<p>
|
|
Truncate large text fields to a configurable limit, appending a size
|
|
hint such as
|
|
<code
|
|
>(truncated, 2847 chars total — use --full to see complete
|
|
body)</code
|
|
>. This prevents a single verbose response from consuming the
|
|
agent’s context budget while preserving enough content for most
|
|
tasks.
|
|
</p>
|
|
|
|
<!-- Discoverability -->
|
|
<hr class="divider" />
|
|
<h3>Discoverability</h3>
|
|
|
|
<h4><span class="text-accent">4.</span> Content first</h4>
|
|
<p>
|
|
Running a command with no arguments should display live, actionable
|
|
data rather than help text.
|
|
</p>
|
|
|
|
<div class="grid grid--2">
|
|
<div class="card">
|
|
<p class="section-label">Conventional: no-args shows help</p>
|
|
<pre><code>$ gh issue
|
|
Work with GitHub issues.
|
|
|
|
USAGE
|
|
gh issue <command> [flags]
|
|
|
|
AVAILABLE COMMANDS
|
|
close, create, list, view, ...</code></pre>
|
|
</div>
|
|
<div class="card">
|
|
<p class="section-label">AXI: no-args shows live state</p>
|
|
<pre><code>$ gh-axi issue
|
|
count: 14 of 8771 total
|
|
issues[14]{number,title,state}:
|
|
51815,"[Bug]: Telegram plugin...",open
|
|
...
|
|
help[2]:
|
|
Run `gh-axi issue view <number>`
|
|
Run `gh-axi issue create --title "..."`</code></pre>
|
|
</div>
|
|
</div>
|
|
|
|
<h4><span class="text-accent">5.</span> Contextual disclosure</h4>
|
|
<p>
|
|
Append <code>help[]</code> lines after output, suggesting logical next
|
|
steps as complete, copy-pasteable commands. This eliminates
|
|
tool-discovery turns and guides agents through multi-step workflows.
|
|
</p>
|
|
|
|
<h4><span class="text-accent">6.</span> Consistent way to get help</h4>
|
|
<p>
|
|
Each subcommand should offer a concise <code>--help</code> flag as a
|
|
fallback when contextual hints are insufficient.
|
|
</p>
|
|
|
|
<!-- Completeness -->
|
|
<hr class="divider" />
|
|
<h3>Completeness</h3>
|
|
|
|
<h4><span class="text-accent">7.</span> Pre-computed fields</h4>
|
|
<p>
|
|
Include aggregate or derived fields that eliminate round trips. The
|
|
most impactful example is <code>totalCount</code>: always report the
|
|
total number of items, not just the page size. Other examples include
|
|
computed CI status summaries (e.g., “27 passed, 0 failed, 10
|
|
skipped”) inline in PR views.
|
|
</p>
|
|
|
|
<div class="grid grid--2">
|
|
<div class="card">
|
|
<p class="section-label">Conventional: no total count</p>
|
|
<pre><code>$ gh label list
|
|
bug Something isn't working #d73a4a
|
|
docs Improvements or additions #0075ca
|
|
... (30 rows -- default page, no total)</code></pre>
|
|
</div>
|
|
<div class="card">
|
|
<p class="section-label">AXI: total count + CI pre-computed</p>
|
|
<pre><code>$ gh-axi label list
|
|
count: 126
|
|
labels[126]{name}:
|
|
bug
|
|
docs
|
|
...
|
|
|
|
$ gh-axi pr view 51772
|
|
pull_request:
|
|
title: "refactor(plugins): route..."
|
|
state: merged
|
|
checks: "27 passed, 0 failed, 10 skipped"</code></pre>
|
|
</div>
|
|
</div>
|
|
|
|
<h4><span class="text-accent">8.</span> Definitive empty states</h4>
|
|
<p>
|
|
When a query returns no results, output an explicit zero-result
|
|
message rather than empty output. Agents cannot distinguish “no
|
|
output” from “command failed silently” without this
|
|
signal.
|
|
</p>
|
|
|
|
<!-- Robustness -->
|
|
<hr class="divider" />
|
|
<h3>Robustness</h3>
|
|
|
|
<h4><span class="text-accent">9.</span> Graceful error handling</h4>
|
|
<p>
|
|
Mutations should be idempotent, errors should be structured and
|
|
written to stdout (not stderr), and commands must never prompt for
|
|
interactive input.
|
|
</p>
|
|
|
|
<h4><span class="text-accent">10.</span> Output discipline</h4>
|
|
<p>
|
|
Reserve stdout for structured data and stderr for debug/log output.
|
|
Use clean exit codes: 0 for success, 1 for errors.
|
|
</p>
|
|
</section>
|
|
|
|
<!-- 4. Experimental Setup -->
|
|
<section class="section">
|
|
<p class="section-label">Method</p>
|
|
<h2>Experimental Setup</h2>
|
|
|
|
<h4>Conditions</h4>
|
|
<p>The benchmark evaluates five agent-tool interface conditions:</p>
|
|
|
|
<div class="card">
|
|
<div class="table-wrap">
|
|
<table class="table">
|
|
<thead>
|
|
<tr>
|
|
<th>Condition</th>
|
|
<th>Interface</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><strong>axi</strong></td>
|
|
<td><code>gh-axi</code></td>
|
|
<td>AXI-compliant wrapper over gh CLI</td>
|
|
</tr>
|
|
<tr>
|
|
<td><strong>cli</strong></td>
|
|
<td><code>gh</code> CLI</td>
|
|
<td>Raw GitHub CLI baseline</td>
|
|
</tr>
|
|
<tr>
|
|
<td><strong>mcp-no-toolsearch</strong></td>
|
|
<td>GitHub MCP</td>
|
|
<td>All tool schemas loaded upfront</td>
|
|
</tr>
|
|
<tr>
|
|
<td><strong>mcp-with-toolsearch</strong></td>
|
|
<td>GitHub MCP + ToolSearch</td>
|
|
<td>Tools discovered on demand via ToolSearch</td>
|
|
</tr>
|
|
<tr>
|
|
<td><strong>mcp-with-code-mode</strong></td>
|
|
<td>TypeScript wrappers over GitHub MCP</td>
|
|
<td>Agent writes <code>.ts</code> scripts to call tools</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
|
|
<h4>Task design</h4>
|
|
<p>The benchmark includes 17 tasks across four categories:</p>
|
|
<ul>
|
|
<li>
|
|
<strong>Simple lookups (4 tasks):</strong>
|
|
<code>list_open_issues</code>, <code>view_pr</code>,
|
|
<code>list_releases</code>, <code>repo_overview</code> —
|
|
single-command tasks.
|
|
</li>
|
|
<li>
|
|
<strong>Multi-step investigations (7 tasks):</strong>
|
|
<code>issue_then_comments</code>, <code>pr_then_checks</code>,
|
|
<code>release_then_body</code>, <code>run_then_jobs</code>,
|
|
<code>ci_failure_investigation</code>, <code>pr_review_prep</code>,
|
|
<code>merged_pr_ci_audit</code> — tasks requiring 2–5
|
|
sequential commands.
|
|
</li>
|
|
<li>
|
|
<strong>Aggregate/count tasks (4 tasks):</strong>
|
|
<code>list_labels</code>, <code>bug_triage_search</code>,
|
|
<code>weekly_catchup</code>, <code>find_fix_for_bug</code> —
|
|
tasks requiring counts or aggregations.
|
|
</li>
|
|
<li>
|
|
<strong>Error handling (2 tasks):</strong>
|
|
<code>invalid_issue</code>, <code>nonexistent_repo</code> —
|
|
tasks where the target does not exist.
|
|
</li>
|
|
</ul>
|
|
|
|
<h4>Infrastructure</h4>
|
|
<p>
|
|
Each run uses a fresh shallow clone of <code>openclaw/openclaw</code>.
|
|
Agent: Claude Sonnet 4.6. Judge: Claude Sonnet 4.6. Five repeats per
|
|
condition–task pair: 17 × 5 × 5 = 425 total runs.
|
|
</p>
|
|
|
|
<h4>Metrics</h4>
|
|
<ul>
|
|
<li>
|
|
<strong>Task success</strong> (binary): determined by an LLM judge
|
|
comparing the agent’s answer against a reference answer and
|
|
rubric.
|
|
</li>
|
|
<li>
|
|
<strong>Cost</strong> (USD): total API cost, computed from
|
|
input/output token counts at published rates.
|
|
</li>
|
|
<li>
|
|
<strong>Duration</strong> (seconds): wall-clock time from task start
|
|
to final response.
|
|
</li>
|
|
<li>
|
|
<strong>Turns</strong>: number of tool invocations the agent makes.
|
|
</li>
|
|
</ul>
|
|
</section>
|
|
|
|
<!-- 5. Results -->
|
|
<section class="section">
|
|
<p class="section-label">Proof</p>
|
|
<h2>Results</h2>
|
|
<p class="text-muted">
|
|
425 runs, 17 tasks × 5 repetitions per condition
|
|
</p>
|
|
|
|
<div class="card mt-xl">
|
|
<p class="section-label">Aggregate</p>
|
|
<div class="table-wrap">
|
|
<table class="table">
|
|
<thead>
|
|
<tr>
|
|
<th>Condition</th>
|
|
<th>Success</th>
|
|
<th>Avg Cost</th>
|
|
<th>Avg Duration</th>
|
|
<th>Avg Turns</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>gh-cli</td>
|
|
<td>86%</td>
|
|
<td>$0.054</td>
|
|
<td>17.4s</td>
|
|
<td>3</td>
|
|
</tr>
|
|
<tr>
|
|
<td>gh-mcp</td>
|
|
<td>87%</td>
|
|
<td>$0.148</td>
|
|
<td>34.2s</td>
|
|
<td>6</td>
|
|
</tr>
|
|
<tr>
|
|
<td>gh-mcp-search</td>
|
|
<td>82%</td>
|
|
<td>$0.147</td>
|
|
<td>41.1s</td>
|
|
<td>8</td>
|
|
</tr>
|
|
<tr>
|
|
<td>gh-mcp-code</td>
|
|
<td>84%</td>
|
|
<td>$0.101</td>
|
|
<td>43.4s</td>
|
|
<td>7</td>
|
|
</tr>
|
|
<tr>
|
|
<td><strong>gh-axi</strong></td>
|
|
<td><strong class="text-accent">100%</strong></td>
|
|
<td><strong class="text-accent">$0.050</strong></td>
|
|
<td><strong class="text-accent">15.7s</strong></td>
|
|
<td><strong class="text-accent">3</strong></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
|
|
<p class="mt-md">
|
|
AXI achieves 100% task success at the lowest average cost
|
|
($0.050/task) and shortest average duration (15.7s). The raw CLI
|
|
baseline achieves 86% success at comparable cost. All three MCP
|
|
conditions achieve 82–87% success at 2–3× the cost
|
|
and 2–3× the duration.
|
|
</p>
|
|
|
|
<hr class="divider" />
|
|
|
|
<h3>Per-task breakdown</h3>
|
|
<p class="text-sm text-muted">
|
|
Success rate (out of 5 runs), average cost, and average duration per
|
|
condition–task pair.
|
|
</p>
|
|
|
|
<div class="card mt-md">
|
|
<div class="table-wrap">
|
|
<table class="table" style="font-size: 0.8rem">
|
|
<thead>
|
|
<tr>
|
|
<th
|
|
rowspan="2"
|
|
style="min-width: 10rem; vertical-align: bottom"
|
|
>
|
|
Task
|
|
</th>
|
|
<th
|
|
colspan="3"
|
|
style="text-align: center; color: var(--color-accent)"
|
|
>
|
|
axi
|
|
</th>
|
|
<th colspan="3" style="text-align: center">cli</th>
|
|
<th colspan="3" style="text-align: center">mcp</th>
|
|
<th colspan="3" style="text-align: center">mcp-search</th>
|
|
<th colspan="3" style="text-align: center">mcp-code</th>
|
|
</tr>
|
|
<tr>
|
|
<th>✓</th>
|
|
<th>$</th>
|
|
<th>⏱</th>
|
|
<th>✓</th>
|
|
<th>$</th>
|
|
<th>⏱</th>
|
|
<th>✓</th>
|
|
<th>$</th>
|
|
<th>⏱</th>
|
|
<th>✓</th>
|
|
<th>$</th>
|
|
<th>⏱</th>
|
|
<th>✓</th>
|
|
<th>$</th>
|
|
<th>⏱</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr class="row-category">
|
|
<td colspan="16"><em>Simple lookups</em></td>
|
|
</tr>
|
|
<tr>
|
|
<td>list_open_issues</td>
|
|
<td>5/5</td>
|
|
<td>.028</td>
|
|
<td>10</td>
|
|
<td>5/5</td>
|
|
<td>.024</td>
|
|
<td>8</td>
|
|
<td>5/5</td>
|
|
<td>.100</td>
|
|
<td>25</td>
|
|
<td>5/5</td>
|
|
<td>.046</td>
|
|
<td>18</td>
|
|
<td>5/5</td>
|
|
<td>.086</td>
|
|
<td>37</td>
|
|
</tr>
|
|
<tr>
|
|
<td>view_pr</td>
|
|
<td>5/5</td>
|
|
<td>.039</td>
|
|
<td>8</td>
|
|
<td>5/5</td>
|
|
<td>.038</td>
|
|
<td>9</td>
|
|
<td>5/5</td>
|
|
<td>.045</td>
|
|
<td>9</td>
|
|
<td>5/5</td>
|
|
<td>.031</td>
|
|
<td>12</td>
|
|
<td>5/5</td>
|
|
<td>.072</td>
|
|
<td>28</td>
|
|
</tr>
|
|
<tr>
|
|
<td>list_releases</td>
|
|
<td>5/5</td>
|
|
<td>.039</td>
|
|
<td>11</td>
|
|
<td>5/5</td>
|
|
<td>.038</td>
|
|
<td>9</td>
|
|
<td>5/5</td>
|
|
<td>.096</td>
|
|
<td>8</td>
|
|
<td>5/5</td>
|
|
<td>.055</td>
|
|
<td>11</td>
|
|
<td>5/5</td>
|
|
<td>.035</td>
|
|
<td>18</td>
|
|
</tr>
|
|
<tr>
|
|
<td>repo_overview</td>
|
|
<td>5/5</td>
|
|
<td>.038</td>
|
|
<td>9</td>
|
|
<td>5/5</td>
|
|
<td>.038</td>
|
|
<td>10</td>
|
|
<td>5/5</td>
|
|
<td>.056</td>
|
|
<td>9</td>
|
|
<td>5/5</td>
|
|
<td>.042</td>
|
|
<td>14</td>
|
|
<td>5/5</td>
|
|
<td>.033</td>
|
|
<td>16</td>
|
|
</tr>
|
|
<tr class="row-category">
|
|
<td colspan="16"><em>Multi-step investigations</em></td>
|
|
</tr>
|
|
<tr>
|
|
<td>issue_then_comments</td>
|
|
<td>5/5</td>
|
|
<td>.049</td>
|
|
<td>13</td>
|
|
<td>5/5</td>
|
|
<td>.046</td>
|
|
<td>11</td>
|
|
<td>5/5</td>
|
|
<td>.065</td>
|
|
<td>14</td>
|
|
<td>5/5</td>
|
|
<td>.050</td>
|
|
<td>20</td>
|
|
<td>5/5</td>
|
|
<td>.090</td>
|
|
<td>39</td>
|
|
</tr>
|
|
<tr>
|
|
<td>pr_then_checks</td>
|
|
<td>5/5</td>
|
|
<td>.050</td>
|
|
<td>14</td>
|
|
<td>5/5</td>
|
|
<td>.055</td>
|
|
<td>15</td>
|
|
<td>5/5</td>
|
|
<td>.112</td>
|
|
<td>32</td>
|
|
<td>5/5</td>
|
|
<td>.077</td>
|
|
<td>35</td>
|
|
<td>5/5</td>
|
|
<td>.087</td>
|
|
<td>46</td>
|
|
</tr>
|
|
<tr>
|
|
<td>release_then_body</td>
|
|
<td>5/5</td>
|
|
<td>.053</td>
|
|
<td>14</td>
|
|
<td>5/5</td>
|
|
<td>.052</td>
|
|
<td>12</td>
|
|
<td>5/5</td>
|
|
<td>.060</td>
|
|
<td>11</td>
|
|
<td>5/5</td>
|
|
<td>.041</td>
|
|
<td>16</td>
|
|
<td>5/5</td>
|
|
<td>.103</td>
|
|
<td>43</td>
|
|
</tr>
|
|
<tr>
|
|
<td>run_then_jobs</td>
|
|
<td>5/5</td>
|
|
<td>.048</td>
|
|
<td>13</td>
|
|
<td>5/5</td>
|
|
<td>.050</td>
|
|
<td>14</td>
|
|
<td>4/5</td>
|
|
<td>.063</td>
|
|
<td>24</td>
|
|
<td class="cell-fail">0/5</td>
|
|
<td>.061</td>
|
|
<td>24</td>
|
|
<td>5/5</td>
|
|
<td>.047</td>
|
|
<td>20</td>
|
|
</tr>
|
|
<tr>
|
|
<td>ci_failure_investigation</td>
|
|
<td>5/5</td>
|
|
<td>.065</td>
|
|
<td>21</td>
|
|
<td>5/5</td>
|
|
<td>.061</td>
|
|
<td>22</td>
|
|
<td class="cell-fail">2/5</td>
|
|
<td>.758</td>
|
|
<td>177</td>
|
|
<td class="cell-fail">2/5</td>
|
|
<td>.568</td>
|
|
<td>151</td>
|
|
<td>5/5</td>
|
|
<td>.194</td>
|
|
<td>94</td>
|
|
</tr>
|
|
<tr>
|
|
<td>pr_review_prep</td>
|
|
<td>5/5</td>
|
|
<td>.055</td>
|
|
<td>19</td>
|
|
<td>5/5</td>
|
|
<td>.070</td>
|
|
<td>25</td>
|
|
<td>5/5</td>
|
|
<td>.100</td>
|
|
<td>32</td>
|
|
<td>5/5</td>
|
|
<td>.087</td>
|
|
<td>31</td>
|
|
<td>5/5</td>
|
|
<td>.091</td>
|
|
<td>35</td>
|
|
</tr>
|
|
<tr>
|
|
<td>merged_pr_ci_audit</td>
|
|
<td>5/5</td>
|
|
<td>.064</td>
|
|
<td>26</td>
|
|
<td>5/5</td>
|
|
<td>.076</td>
|
|
<td>42</td>
|
|
<td>3/5</td>
|
|
<td>.340</td>
|
|
<td>63</td>
|
|
<td>3/5</td>
|
|
<td>.614</td>
|
|
<td>136</td>
|
|
<td>5/5</td>
|
|
<td>.205</td>
|
|
<td>104</td>
|
|
</tr>
|
|
<tr class="row-category">
|
|
<td colspan="16"><em>Aggregate/count tasks</em></td>
|
|
</tr>
|
|
<tr>
|
|
<td>list_labels</td>
|
|
<td>5/5</td>
|
|
<td>.044</td>
|
|
<td>11</td>
|
|
<td class="cell-fail">0/5</td>
|
|
<td>.041</td>
|
|
<td>10</td>
|
|
<td class="cell-fail">0/5</td>
|
|
<td>.319</td>
|
|
<td>108</td>
|
|
<td class="cell-fail">0/5</td>
|
|
<td>.353</td>
|
|
<td>125</td>
|
|
<td class="cell-fail">0/5</td>
|
|
<td>.041</td>
|
|
<td>18</td>
|
|
</tr>
|
|
<tr>
|
|
<td>bug_triage_search</td>
|
|
<td>5/5</td>
|
|
<td>.088</td>
|
|
<td>47</td>
|
|
<td>3/5</td>
|
|
<td>.114</td>
|
|
<td>52</td>
|
|
<td>5/5</td>
|
|
<td>.092</td>
|
|
<td>21</td>
|
|
<td>5/5</td>
|
|
<td>.088</td>
|
|
<td>19</td>
|
|
<td>5/5</td>
|
|
<td>.132</td>
|
|
<td>53</td>
|
|
</tr>
|
|
<tr>
|
|
<td>weekly_catchup</td>
|
|
<td>5/5</td>
|
|
<td>.050</td>
|
|
<td>15</td>
|
|
<td class="cell-fail">0/5</td>
|
|
<td>.046</td>
|
|
<td>14</td>
|
|
<td>5/5</td>
|
|
<td>.123</td>
|
|
<td>17</td>
|
|
<td>5/5</td>
|
|
<td>.185</td>
|
|
<td>37</td>
|
|
<td class="cell-fail">0/5</td>
|
|
<td>.132</td>
|
|
<td>53</td>
|
|
</tr>
|
|
<tr>
|
|
<td>find_fix_for_bug</td>
|
|
<td>5/5</td>
|
|
<td>.067</td>
|
|
<td>21</td>
|
|
<td>5/5</td>
|
|
<td>.091</td>
|
|
<td>26</td>
|
|
<td>5/5</td>
|
|
<td>.084</td>
|
|
<td>15</td>
|
|
<td>5/5</td>
|
|
<td>.093</td>
|
|
<td>29</td>
|
|
<td>5/5</td>
|
|
<td>.124</td>
|
|
<td>43</td>
|
|
</tr>
|
|
<tr class="row-category">
|
|
<td colspan="16"><em>Error handling</em></td>
|
|
</tr>
|
|
<tr>
|
|
<td>invalid_issue</td>
|
|
<td>5/5</td>
|
|
<td>.038</td>
|
|
<td>8</td>
|
|
<td>5/5</td>
|
|
<td>.038</td>
|
|
<td>8</td>
|
|
<td>5/5</td>
|
|
<td>.053</td>
|
|
<td>8</td>
|
|
<td>5/5</td>
|
|
<td>.051</td>
|
|
<td>10</td>
|
|
<td>5/5</td>
|
|
<td>.167</td>
|
|
<td>69</td>
|
|
</tr>
|
|
<tr>
|
|
<td>nonexistent_repo</td>
|
|
<td>5/5</td>
|
|
<td>.039</td>
|
|
<td>8</td>
|
|
<td>5/5</td>
|
|
<td>.039</td>
|
|
<td>10</td>
|
|
<td>5/5</td>
|
|
<td>.052</td>
|
|
<td>8</td>
|
|
<td>5/5</td>
|
|
<td>.050</td>
|
|
<td>10</td>
|
|
<td class="cell-fail">1/5</td>
|
|
<td>.069</td>
|
|
<td>21</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
<p class="text-sm text-muted mt-sm">
|
|
✓ = successes/5 runs. $ = avg cost (USD). ⏱ = avg
|
|
duration (seconds).
|
|
</p>
|
|
</div>
|
|
</section>
|
|
|
|
<!-- 6. Analysis -->
|
|
<section class="section">
|
|
<p class="section-label">Analysis</p>
|
|
<h2>Findings</h2>
|
|
|
|
<div class="card mt-xl">
|
|
<p class="section-label">Finding 1</p>
|
|
<h4>AXI achieves 100% reliability at lowest cost.</h4>
|
|
<p class="mb-0">
|
|
AXI is the only condition that passes all 85 runs (100% success) at
|
|
an average cost of $0.050/task — 7% less than CLI’s
|
|
$0.054 and 66% less than the cheapest MCP condition ($0.101 for
|
|
<code>mcp-with-code-mode</code>). AXI also has the fastest average
|
|
duration (15.7s vs. 17.4s for CLI, 34–43s for MCP). The
|
|
reliability advantage is most pronounced on aggregate tasks (where
|
|
pre-computed <code>totalCount</code> fields eliminate pagination
|
|
errors) and complex investigations (where contextual suggestions
|
|
prevent wrong turns).
|
|
</p>
|
|
</div>
|
|
|
|
<div class="card mt-md">
|
|
<p class="section-label">Finding 2</p>
|
|
<h4>MCP conditions are 2–3× more expensive.</h4>
|
|
<p>
|
|
The MCP cost premium is not uniform — it concentrates on
|
|
complex tasks where schema overhead compounds across many turns. On
|
|
<code>ci_failure_investigation</code>,
|
|
<code>mcp-no-toolsearch</code> averages $0.758/task across 15 turns
|
|
and 177 seconds, vs. AXI’s $0.065 across 3 turns and 21
|
|
seconds — a <strong>12× cost difference</strong>. The
|
|
agent re-sends the full tool schema catalog (176K tokens) on every
|
|
turn; by turn 15, schema tokens dominate the context.
|
|
</p>
|
|
<p class="mb-0">
|
|
On simple tasks where agents need few turns, MCP’s overhead is
|
|
more modest: <code>view_pr</code> costs $0.045 for MCP vs. $0.039
|
|
for AXI (1.2×).
|
|
</p>
|
|
</div>
|
|
|
|
<div class="card mt-md">
|
|
<p class="section-label">Finding 3</p>
|
|
<h4>
|
|
ToolSearch saves upfront tokens but spends them on extra turns.
|
|
</h4>
|
|
<p>
|
|
ToolSearch starts with a smaller context but needs extra turns for
|
|
tool discovery. Whether that trade-off pays depends on task
|
|
complexity:
|
|
</p>
|
|
<div class="table-wrap">
|
|
<table class="table" style="font-size: 0.875rem">
|
|
<thead>
|
|
<tr>
|
|
<th></th>
|
|
<th colspan="4" style="text-align: center">mcp-search</th>
|
|
<th colspan="4" style="text-align: center">mcp (raw)</th>
|
|
</tr>
|
|
<tr>
|
|
<th>Task</th>
|
|
<th>1st turn</th>
|
|
<th>last turn</th>
|
|
<th>turns</th>
|
|
<th>cost</th>
|
|
<th>1st turn</th>
|
|
<th>last turn</th>
|
|
<th>turns</th>
|
|
<th>cost</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td><code>view_pr</code></td>
|
|
<td>16K</td>
|
|
<td>18K</td>
|
|
<td>3</td>
|
|
<td>$0.031</td>
|
|
<td>41K</td>
|
|
<td>42K</td>
|
|
<td>2</td>
|
|
<td>$0.045</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>find_fix_for_bug</code></td>
|
|
<td>16K</td>
|
|
<td>22K</td>
|
|
<td>6</td>
|
|
<td>$0.093</td>
|
|
<td>42K</td>
|
|
<td>45K</td>
|
|
<td>3</td>
|
|
<td>$0.084</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>weekly_catchup</code></td>
|
|
<td>16K</td>
|
|
<td>43K</td>
|
|
<td>6</td>
|
|
<td>$0.185</td>
|
|
<td>42K</td>
|
|
<td>59K</td>
|
|
<td>4</td>
|
|
<td>$0.123</td>
|
|
</tr>
|
|
<tr>
|
|
<td><code>merged_pr_ci_audit</code></td>
|
|
<td>16K</td>
|
|
<td>84K</td>
|
|
<td>24</td>
|
|
<td>$0.614</td>
|
|
<td>42K</td>
|
|
<td>85K</td>
|
|
<td>18</td>
|
|
<td>$0.340</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
<p>
|
|
Both conditions start at a fixed context size — 16K for
|
|
ToolSearch (no schemas), 42K for raw MCP (all schemas loaded)
|
|
— and grow as conversation history accumulates. On a simple
|
|
<code>view_pr</code>, ToolSearch finishes at 18K (still well under
|
|
raw’s 42K), so the upfront savings carry through to a lower
|
|
cost despite one extra turn.
|
|
</p>
|
|
<p>
|
|
But as tasks grow, the contexts converge. By
|
|
<code>weekly_catchup</code>, ToolSearch’s last turn has
|
|
reached 43K — matching raw’s <em>starting</em> context
|
|
— yet it still needs 2 extra turns, costing 1.5× more.
|
|
On <code>merged_pr_ci_audit</code>, both end at ~85K tokens, but
|
|
ToolSearch takes 24 turns to get there vs. 18 — the upfront
|
|
savings are completely gone, and the extra turns make it 1.8×
|
|
more expensive.
|
|
</p>
|
|
<p>
|
|
ToolSearch also introduces a new failure mode: on
|
|
<code>run_then_jobs</code>, the agent cannot find the right tool via
|
|
search (0/5 success), while raw loading succeeds 4/5.
|
|
</p>
|
|
<p class="mb-0">
|
|
The takeaway: lazy tool loading helps on simple tasks where one
|
|
discovery turn unlocks big token savings. But on complex tasks where
|
|
the agent needs many tools across many turns, the discovery overhead
|
|
compounds and overwhelms the savings. Since real-world agent
|
|
workloads skew complex, raw loading is the safer default.
|
|
</p>
|
|
</div>
|
|
|
|
<div class="card mt-md">
|
|
<p class="section-label">Finding 4</p>
|
|
<h4>Code-mode is cheapest MCP but still 2× AXI.</h4>
|
|
<p>
|
|
The <code>mcp-with-code-mode</code> condition, where the agent
|
|
writes TypeScript scripts calling typed wrapper functions, achieves
|
|
the lowest cost among MCP conditions ($0.101/task). Writing code
|
|
amortizes schema costs by batching multiple API calls per script.
|
|
But the code-generation overhead makes it the
|
|
<em>slowest</em> condition (43.4s avg, the longest of all five).
|
|
</p>
|
|
<p class="mb-0">
|
|
On <code>ci_failure_investigation</code>,
|
|
<code>mcp-with-code-mode</code> costs $0.194/task — 3×
|
|
AXI’s $0.065 — but achieves 5/5 success, outperforming
|
|
both direct MCP conditions (2/5 each). Code-mode also introduces a
|
|
unique failure mode: unhandled runtime errors cause 4/5 failures on
|
|
<code>nonexistent_repo</code>, where the agent writes scripts that
|
|
throw instead of gracefully reporting the error.
|
|
</p>
|
|
</div>
|
|
|
|
<div class="card mt-md">
|
|
<p class="section-label">Finding 5</p>
|
|
<h4>AXI’s advantages by task type.</h4>
|
|
<p>
|
|
<strong>Simple lookups.</strong> All conditions achieve 100%
|
|
success. AXI and CLI cost $0.024–0.039/task; MCP costs
|
|
$0.031–0.100/task. The 1.5–3× MCP premium is pure
|
|
schema overhead.
|
|
</p>
|
|
<p>
|
|
<strong>Multi-step investigations.</strong> AXI and CLI both achieve
|
|
100% success on most tasks. The cost difference emerges on the
|
|
hardest tasks: <code>ci_failure_investigation</code> costs $0.065
|
|
for AXI vs. $0.061 for CLI (both 5/5), but $0.194–0.758 for
|
|
MCP conditions (2–5/5). AXI’s pre-computed job status
|
|
fields (<code>checks: "27 passed, 0 failed"</code>)
|
|
eliminate the need for separate job-listing API calls that MCP
|
|
agents must make turn by turn.
|
|
</p>
|
|
<p>
|
|
<strong>Aggregate tasks.</strong> This is where AXI separates from
|
|
all conditions. <code>list_labels</code>: AXI 5/5 at $0.044; every
|
|
other condition 0/5. <code>weekly_catchup</code>: AXI 5/5 at $0.050;
|
|
CLI 0/5; <code>mcp-with-code-mode</code> 0/5. The pre-computed
|
|
<code>totalCount</code> field is the difference — without it,
|
|
agents report the default page size (30) as the total.
|
|
</p>
|
|
<p class="mb-0">
|
|
<strong>Error handling.</strong> All CLI-based conditions handle
|
|
errors well. <code>mcp-with-code-mode</code> scores only 1/5 on
|
|
<code>nonexistent_repo</code> ($0.069/task) because generated
|
|
scripts throw unhandled exceptions rather than reporting structured
|
|
errors.
|
|
</p>
|
|
</div>
|
|
</section>
|
|
|
|
<!-- 7. Case Studies -->
|
|
<section class="section">
|
|
<p class="section-label">Case Studies</p>
|
|
<h2>Trajectory analysis.</h2>
|
|
|
|
<h3>Case A: ci_failure_investigation</h3>
|
|
<p class="text-muted">
|
|
Identify the 5 most recent failed CI runs and report which jobs failed
|
|
in each.
|
|
</p>
|
|
|
|
<div class="grid grid--2 mt-md">
|
|
<div class="card">
|
|
<p class="section-label">
|
|
gh-axi — 3 turns · $0.065 · 21s · 5/5
|
|
</p>
|
|
<pre><code><span class="annotation">Turn 1 (agent → tool):</span>
|
|
$ gh-axi run list --status failure --limit 5
|
|
|
|
<span class="annotation">Turn 1 (tool → agent):</span>
|
|
runs[5]{id,title,status,conclusion,workflow,branch}:
|
|
23386042824,"fix(daemon): repair stale...",completed,failure,Install Smoke,...
|
|
23385977677,"feat(auth): add models auth...",completed,failure,CI,...
|
|
... (3 more)
|
|
help[1]:
|
|
Run `gh-axi run view <id>` to view details
|
|
|
|
<span class="annotation">Turn 2: Agent views all 5 runs in parallel</span>
|
|
|
|
<span class="annotation">Turn 2 (tool → agent, for each run):</span>
|
|
run:
|
|
id: 23385977677
|
|
conclusion: failure
|
|
workflow: CI
|
|
jobs[29]{name,status,conclusion}:
|
|
check,completed,failure
|
|
check-additional,completed,failure
|
|
build-smoke,completed,failure
|
|
... (26 more with pass/skip/fail)
|
|
help[2]:
|
|
Run `gh-axi run view 23385977677 --log-failed`
|
|
|
|
<span class="annotation">Turn 3: Agent summarizes patterns.</span></code></pre>
|
|
</div>
|
|
<div class="card">
|
|
<p class="section-label">
|
|
gh mcp (raw) — 15 turns · $0.758 · 177s
|
|
· 2/5
|
|
</p>
|
|
<pre><code><span class="annotation">Turn 1: list_workflow_runs(status:"failure",per_page:5)
|
|
→ 5 run objects (~2KB JSON each)</span>
|
|
|
|
<span class="annotation">Turn 2: list_workflow_run_jobs(run_id:...)
|
|
→ 29 job objects (~1.5KB JSON each)
|
|
(context now: 176K schema + growing conversation)</span>
|
|
|
|
<span class="annotation">Turn 3: list_workflow_run_jobs(run_id:...)
|
|
→ 2 job objects</span>
|
|
|
|
<span class="annotation">Turn 4–6: ... (remaining runs)
|
|
(context exceeding 400K tokens by turn 6)</span>
|
|
|
|
<span class="annotation">Turn 7–15: Agent re-reads earlier results,
|
|
attempts to correlate, loses track of which
|
|
jobs belong to which run.</span>
|
|
|
|
<span class="annotation">Result: 2/5 runs produce correct answer.
|
|
3/5 runs misattribute job failures or omit runs.</span></code></pre>
|
|
</div>
|
|
</div>
|
|
<p class="mt-md">
|
|
AXI completes in 3 turns. The key advantage: each
|
|
<code>run view</code> response includes the full job-level breakdown
|
|
(<code>jobs[29]{name,status,conclusion}</code>) as structured data,
|
|
plus a suggestion to view failure logs. The agent can immediately
|
|
identify which jobs failed without additional API calls. MCP requires
|
|
separate API calls for run listing and job listing (no inline job
|
|
status). Each call re-sends the full schema catalog. By turn 6, the
|
|
context exceeds 400K tokens, and the agent begins confusing job data
|
|
across runs.
|
|
</p>
|
|
|
|
<hr class="divider" />
|
|
|
|
<h3>Case B: merged_pr_ci_audit</h3>
|
|
<p class="text-muted">
|
|
List the 10 most recently merged PRs and report the CI status of each.
|
|
</p>
|
|
|
|
<div class="card mt-md">
|
|
<p class="section-label">
|
|
gh-axi — 5 turns · $0.064 · 26s · 5/5
|
|
</p>
|
|
<pre><code><span class="annotation">Turn 1:</span>
|
|
$ gh-axi pr list --state merged --limit 10
|
|
count: 10 of 3549
|
|
pull_requests[10]{number,title,state,...}:
|
|
51772,"refactor(plugins): route Telegram...",merged
|
|
...
|
|
help: Run `gh-axi pr view <number>`
|
|
|
|
<span class="annotation">Turns 2–5: views each PR (some in parallel)</span>
|
|
pr:
|
|
number: 51772
|
|
state: merged
|
|
checks: "27 passed, 0 failed, 10 skipped"
|
|
<span class="annotation"> ^ pre-computed — no extra API call</span>
|
|
|
|
<span class="annotation">Agent summarizes: “6 out of 10 fully green CI.”</span></code></pre>
|
|
</div>
|
|
|
|
<div class="card mt-md">
|
|
<p class="section-label">
|
|
gh CLI — 4 turns · $0.076 · 42s · 5/5
|
|
</p>
|
|
<pre><code><span class="annotation">Turn 1:</span>
|
|
$ gh pr list --state merged --limit 10 \
|
|
--json number,title,author,mergedAt,statusCheckRollup
|
|
→ 10 PR objects with nested check arrays (~2KB JSON each)
|
|
|
|
<span class="annotation">Turns 2–3: parse JSON, count pass/fail per PR</span>
|
|
Some runs call `gh pr checks` again to double-check
|
|
|
|
<span class="annotation">Turn 4: Agent summarizes results.</span>
|
|
<span class="annotation">19% more expensive, 62% slower — but succeeds.</span></code></pre>
|
|
</div>
|
|
|
|
<div class="card mt-md">
|
|
<p class="section-label">
|
|
gh mcp (with ToolSearch) — 24 turns · $0.614 ·
|
|
136s · 3/5
|
|
</p>
|
|
<pre><code><span class="annotation">Turns 1–3: ToolSearch for PR listing tool</span>
|
|
<span class="annotation">Turns 4–6: list merged PRs, paginate</span>
|
|
<span class="annotation">Turns 7–24: for each PR, search for check/status
|
|
tool, invoke it, parse response.
|
|
Context grows to >300K tokens.
|
|
Agent loses track of PR-to-check mapping.</span>
|
|
|
|
<span class="annotation">Result: 3/5 runs succeed. 2/5 misreport CI status.
|
|
9.6× more expensive than AXI.</span></code></pre>
|
|
</div>
|
|
|
|
<p class="mt-md">
|
|
AXI’s pre-computed <code>checks</code> field provides a one-line
|
|
CI summary inline in each PR view. The agent never needs to call a
|
|
separate checks or workflow API. CLI also succeeds (5/5) by using the
|
|
<code>--json</code> flag to get structured output in a single call,
|
|
but costs 19% more ($0.076 vs. $0.064) and takes 62% longer (42s vs.
|
|
26s). The CLI agent must know to use <code>--json</code> with
|
|
<code>statusCheckRollup</code> — a non-obvious field name that
|
|
requires prior knowledge of the <code>gh</code> CLI’s JSON
|
|
schema.
|
|
</p>
|
|
</section>
|
|
|
|
<!-- 8. Limitations -->
|
|
<section class="section">
|
|
<p class="section-label">Limitations</p>
|
|
<h2>Caveats.</h2>
|
|
<ul>
|
|
<li>
|
|
<strong>Single target repository.</strong> All tasks target
|
|
<code>openclaw/openclaw</code>. Results may vary on repositories
|
|
with different sizes or structures.
|
|
</li>
|
|
<li>
|
|
<strong>Single agent model.</strong> Only Claude Sonnet 4.6 is
|
|
evaluated. Other models may exhibit different sensitivities to
|
|
output format.
|
|
</li>
|
|
<li>
|
|
<strong>Read-only tasks.</strong> The benchmark includes only read
|
|
operations. Mutations introduce additional challenges around
|
|
idempotency.
|
|
</li>
|
|
<li>
|
|
<strong>LLM judge.</strong> Task success is determined by an LLM
|
|
judge, which may introduce scoring biases.
|
|
</li>
|
|
<li>
|
|
<strong>GitHub-specific implementation.</strong> While the AXI
|
|
principles are domain-general, this evaluation is specific to
|
|
GitHub.
|
|
</li>
|
|
<li>
|
|
<strong>LLM training data bias.</strong> GitHub CLI is widely
|
|
visible in LLM training data, allowing it to be more efficient than
|
|
a brand new CLI. If there’s enough interest, I can run another
|
|
benchmark removing this variable.
|
|
</li>
|
|
<li>
|
|
<strong>Sub-optimal MCP baseline.</strong> The MCP conditions use
|
|
the official GitHub MCP server, not a hand-optimized one. A
|
|
carefully designed MCP server with fewer, better-composed tools
|
|
could narrow the gap.
|
|
</li>
|
|
</ul>
|
|
</section>
|
|
|
|
<!-- 9. Conclusion -->
|
|
<section class="section">
|
|
<p class="section-label">Implication</p>
|
|
<h2 data-annotate>
|
|
The answer is [principled design], not protocol choice.
|
|
</h2>
|
|
<p>
|
|
The debate between CLI and MCP as agent-tool interfaces misses the
|
|
deeper question: what design principles make any interface effective
|
|
for agents? This evaluation shows that a principled CLI design (AXI)
|
|
outperforms both raw CLI and MCP on every metric — success,
|
|
cost, duration, and turns.
|
|
</p>
|
|
<p>
|
|
AXI achieves 100% reliability at $0.050/task, compared to 86% at
|
|
$0.054 for raw CLI and 82–87% at $0.101–0.148 for MCP. The
|
|
cost gap widens dramatically on complex tasks:
|
|
<code>ci_failure_investigation</code> costs $0.065 for AXI vs. $0.758
|
|
for MCP (12×), with AXI achieving 100% success vs. MCP’s
|
|
40%.
|
|
</p>
|
|
<p>
|
|
The 10 AXI principles provide a concrete, testable framework for
|
|
designing agent-ergonomic interfaces that are both reliable and
|
|
cost-efficient.
|
|
</p>
|
|
</section>
|
|
|
|
<!-- References -->
|
|
<footer
|
|
class="section"
|
|
id="references"
|
|
style="padding-bottom: var(--space-xl)"
|
|
>
|
|
<p class="section-label">References</p>
|
|
<ol>
|
|
<li id="ref-mcp">
|
|
<a href="https://modelcontextprotocol.io"
|
|
>Anthropic. Model Context Protocol specification.</a
|
|
>
|
|
Technical report, 2024.
|
|
</li>
|
|
<li id="ref-smithery">
|
|
<a href="https://smithery.ai/blog/mcp-vs-cli-is-the-wrong-fight"
|
|
>H. Mao and R. Pradhan. “MCP vs CLI is the wrong
|
|
fight.”</a
|
|
>
|
|
Smithery Blog, March 2026.
|
|
</li>
|
|
<li id="ref-anthropic-code">
|
|
<a
|
|
href="https://www.anthropic.com/engineering/code-execution-with-mcp"
|
|
>A. Jones and C. Kelly. “Code execution with MCP: Building
|
|
more efficient agents.”</a
|
|
>
|
|
Anthropic Engineering Blog, November 2025.
|
|
</li>
|
|
<li id="ref-cloudflare-code">
|
|
<a href="https://blog.cloudflare.com/code-mode/"
|
|
>K. Varda and S. Pai. “Code Mode: The better way to use
|
|
MCP.”</a
|
|
>
|
|
Cloudflare Blog, September 2025.
|
|
</li>
|
|
</ol>
|
|
<p class="text-sm text-muted mt-xl">
|
|
<a href="https://github.com/kunchenguid/axi" target="_blank"
|
|
>GitHub Repo</a
|
|
>
|
|
· Kun Chen Presents
|
|
</p>
|
|
</footer>
|
|
</div>
|
|
<script>
|
|
function copyBlock(btn) {
|
|
const code = btn.closest("pre").querySelector("code");
|
|
const copyIcon = btn.querySelector(".icon-copy");
|
|
const checkIcon = btn.querySelector(".icon-check");
|
|
navigator.clipboard.writeText(code.textContent).then(() => {
|
|
copyIcon.style.display = "none";
|
|
checkIcon.style.display = "block";
|
|
btn.style.color = "var(--color-accent)";
|
|
setTimeout(() => {
|
|
copyIcon.style.display = "block";
|
|
checkIcon.style.display = "none";
|
|
btn.style.color = "";
|
|
}, 1500);
|
|
});
|
|
}
|
|
</script>
|
|
</body>
|
|
</html>
|