mirror of
https://github.com/callstack/agent-device.git
synced 2026-09-14 20:06:34 +08:00
Michał Pierzchała
45cfad5cc5
* feat: add e2e command perf benchmark harness + nightly CI Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown reports. Per-command timing comes from wrapping each batchable command in its own single-step batch (daemon durationMs) plus wall-clock around the process. Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and runs the CLI from source via --experimental-strip-types (no dist build). * refactor(perf): drive the harness CLI via runCmdSync, not spawnSync Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not node:child_process directly. Switch the perf harness's invokeCli to runCmdSync (allowFailure so non-zero exits are recorded as samples) and add a maxBuffer option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default). * perf(harness): warm the runner after open so the first measured command is clean The first interaction after open/relaunch pays the one-time iOS XCUITest runner startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was landing on the first measured command each round (snapshot -i), inflating it ~10x vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after each round's reset-open, and after every freshRoot relaunch, so no measured command absorbs runner startup. Noted in the report header. * refactor(perf): address review + fix Fallow CI - exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer change pulled exec.ts into the audit scope). - .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/ test-app) so its naturally-moderate functions don't trip the complexity gate. - config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so --n/--rounds/--warmup report the actual flag + reject non-integers clearly. - harness.ts: extract toSample(); type sampleError param as CliResult. - scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/ step.args ?? []). - comment/legend rewords (platform defaults are local-convenience/CI-overridden; elements = node count). check:fallow now green; typecheck/lint/unit pass. * perf(harness): downgrade sample ok when a batch step reports ok:false Defensive belt-and-suspenders for the Codex review note: stop-only batch already surfaces a failed step as a top-level failure (caught by invokeCli), but if an on-error=continue mode ever keeps the batch ok while a step fails, don't silently count that step as a successful sample — derive ok from the step's own result.ok.
68 lines
3.0 KiB
TypeScript
68 lines
3.0 KiB
TypeScript
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
import type { Measurement, RunResult, Stat } from './types.ts';
|
||
|
||
function ms(n: number | undefined): string {
|
||
return typeof n === 'number' && Number.isFinite(n) ? n.toFixed(0) : '–';
|
||
}
|
||
|
||
function wallCells(s: Stat | null): string {
|
||
if (!s) return '– | – | – | –';
|
||
return `${ms(s.min)} | ${ms(s.median)} | ${ms(s.p95)} | ${ms(s.max)}`;
|
||
}
|
||
|
||
function stampName(platform: string, startedAt: string): string {
|
||
return `perf-${platform}-${startedAt.replace(/[:.]/g, '-')}`;
|
||
}
|
||
|
||
function measurementRow(m: Measurement): string {
|
||
const daemon = m.daemonDuration ? ms(m.daemonDuration.median) : '–';
|
||
const elements = m.elementCount ? ms(m.elementCount.median) : '–';
|
||
const n = m.wallClock?.n ?? 0;
|
||
return `| ${m.label} | ${m.command} | ${m.execMode} | ${n} | ${wallCells(m.wallClock)} | ${daemon} | ${elements} | ${m.notes.join('; ')} |`;
|
||
}
|
||
|
||
function toMarkdown(run: RunResult): string {
|
||
const lines: string[] = [];
|
||
lines.push(`# agent-device command perf — ${run.platform}`);
|
||
lines.push('');
|
||
lines.push(`- **Device**: ${run.device.name} (${run.device.udid ?? run.device.serial ?? '?'})`);
|
||
lines.push(`- **agent-device**: ${run.agentDeviceVersion}`);
|
||
lines.push(`- **Rounds**: ${run.config.rounds} (warmup ${run.config.warmup} dropped)`);
|
||
lines.push(`- **Started**: ${run.startedAt}`);
|
||
lines.push(`- **Finished**: ${run.finishedAt}`);
|
||
lines.push('');
|
||
lines.push('All times in milliseconds. `wall-clock` includes process spawn + socket overhead;');
|
||
lines.push('`daemon` is the batch step round-trip (spawn overhead ≈ wall-median − daemon-median).');
|
||
lines.push('`elements` = node count in the snapshot payload (tree-size proxy).');
|
||
lines.push('An untimed warmup interaction runs after each open/relaunch, so measured commands');
|
||
lines.push('do not pay the one-time iOS-runner startup or post-relaunch first-AX-query cost.');
|
||
lines.push('');
|
||
lines.push('| command | cli | mode | n | wall min | wall median | wall p95 | wall max | daemon median | elements | notes |');
|
||
lines.push('|---|---|---|---|---|---|---|---|---|---|---|');
|
||
for (const m of run.measurements) lines.push(measurementRow(m));
|
||
lines.push('');
|
||
|
||
const failed = run.measurements.filter((m) => m.failures > 0);
|
||
if (failed.length > 0) {
|
||
lines.push('## Failures');
|
||
lines.push('');
|
||
for (const m of failed) {
|
||
const sample = m.samples.find((s) => !s.ok);
|
||
lines.push(`- **${m.label}** — ${m.notes.join('; ')}${sample?.errorMessage ? ` — ${sample.errorMessage}` : ''}`);
|
||
}
|
||
lines.push('');
|
||
}
|
||
return lines.join('\n');
|
||
}
|
||
|
||
export function writeReports(run: RunResult, outDir: string): { jsonPath: string; mdPath: string } {
|
||
fs.mkdirSync(outDir, { recursive: true });
|
||
const base = stampName(run.platform, run.startedAt);
|
||
const jsonPath = path.join(outDir, `${base}.json`);
|
||
const mdPath = path.join(outDir, `${base}.md`);
|
||
fs.writeFileSync(jsonPath, JSON.stringify(run, null, 2));
|
||
fs.writeFileSync(mdPath, toMarkdown(run));
|
||
return { jsonPath, mdPath };
|
||
}
|