mirror of
https://github.com/callstack/agent-device.git
synced 2026-09-14 20:06:34 +08:00
45cfad5cc5
* feat: add e2e command perf benchmark harness + nightly CI Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown reports. Per-command timing comes from wrapping each batchable command in its own single-step batch (daemon durationMs) plus wall-clock around the process. Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and runs the CLI from source via --experimental-strip-types (no dist build). * refactor(perf): drive the harness CLI via runCmdSync, not spawnSync Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not node:child_process directly. Switch the perf harness's invokeCli to runCmdSync (allowFailure so non-zero exits are recorded as samples) and add a maxBuffer option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default). * perf(harness): warm the runner after open so the first measured command is clean The first interaction after open/relaunch pays the one-time iOS XCUITest runner startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was landing on the first measured command each round (snapshot -i), inflating it ~10x vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after each round's reset-open, and after every freshRoot relaunch, so no measured command absorbs runner startup. Noted in the report header. * refactor(perf): address review + fix Fallow CI - exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer change pulled exec.ts into the audit scope). - .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/ test-app) so its naturally-moderate functions don't trip the complexity gate. - config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so --n/--rounds/--warmup report the actual flag + reject non-integers clearly. - harness.ts: extract toSample(); type sampleError param as CliResult. - scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/ step.args ?? []). - comment/legend rewords (platform defaults are local-convenience/CI-overridden; elements = node count). check:fallow now green; typecheck/lint/unit pass. * perf(harness): downgrade sample ok when a batch step reports ok:false Defensive belt-and-suspenders for the Codex review note: stop-only batch already surfaces a failed step as a top-level failure (caught by invokeCli), but if an on-error=continue mode ever keeps the batch ok while a step fails, don't silently count that step as a successful sample — derive ok from the step's own result.ok.
95 lines
3.0 KiB
TypeScript
95 lines
3.0 KiB
TypeScript
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import type { Platform } from './types.ts';
|
|
|
|
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
export const REPO_ROOT = path.resolve(HERE, '..', '..');
|
|
const CLI_BIN = path.join(REPO_ROOT, 'bin', 'agent-device.mjs');
|
|
const DEFAULT_OUT_DIR = path.join(HERE, '.results');
|
|
|
|
export type PerfConfig = {
|
|
platform: Platform;
|
|
rounds: number; // measured rounds (samples per command)
|
|
warmup: number; // leading rounds dropped from stats
|
|
keepArtifacts: boolean; // keep temp state dir + leave device booted
|
|
outDir: string;
|
|
udid?: string; // iOS device override (UDID)
|
|
device?: string; // device override by name (e.g. "iPhone 17 Pro"); preferred over udid
|
|
serial?: string; // Android device override
|
|
};
|
|
|
|
// How to invoke the CLI. Defaults to the built dist binary (bin/agent-device.mjs).
|
|
// Set AGENT_DEVICE_PERF_CLI to run from source instead, e.g. on CI:
|
|
// AGENT_DEVICE_PERF_CLI="--experimental-strip-types src/bin.ts"
|
|
// (matches the device workflows, which run from source and skip the dist build).
|
|
export function resolveCliArgv(): string[] {
|
|
const override = process.env.AGENT_DEVICE_PERF_CLI?.trim();
|
|
if (override) return override.split(/\s+/);
|
|
return [CLI_BIN];
|
|
}
|
|
|
|
export function usesSourceCli(): boolean {
|
|
return Boolean(process.env.AGENT_DEVICE_PERF_CLI?.trim());
|
|
}
|
|
|
|
function readValue(argv: string[], i: number, flag: string): string {
|
|
const v = argv[i + 1];
|
|
if (v === undefined) throw new Error(`Missing value for ${flag}`);
|
|
return v;
|
|
}
|
|
|
|
function readIntValue(argv: string[], i: number, flag: string, min: number): number {
|
|
const raw = readValue(argv, i, flag);
|
|
const n = Number(raw);
|
|
if (!Number.isInteger(n) || n < min) {
|
|
throw new Error(`${flag} must be an integer >= ${min} (got ${JSON.stringify(raw)})`);
|
|
}
|
|
return n;
|
|
}
|
|
|
|
export function parseConfig(argv: string[]): PerfConfig {
|
|
const cfg: PerfConfig = {
|
|
platform: 'ios',
|
|
rounds: 5,
|
|
warmup: 1,
|
|
keepArtifacts: false,
|
|
outDir: DEFAULT_OUT_DIR,
|
|
};
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const a = argv[i];
|
|
switch (a) {
|
|
case '--platform': {
|
|
const v = readValue(argv, i++, a);
|
|
if (v !== 'ios' && v !== 'android') throw new Error(`Unknown platform: ${v}`);
|
|
cfg.platform = v;
|
|
break;
|
|
}
|
|
case '--n':
|
|
case '--rounds':
|
|
cfg.rounds = readIntValue(argv, i++, a, 1);
|
|
break;
|
|
case '--warmup':
|
|
cfg.warmup = readIntValue(argv, i++, a, 0);
|
|
break;
|
|
case '--keep-artifacts':
|
|
cfg.keepArtifacts = true;
|
|
break;
|
|
case '--out-dir':
|
|
cfg.outDir = path.resolve(readValue(argv, i++, a));
|
|
break;
|
|
case '--udid':
|
|
cfg.udid = readValue(argv, i++, a);
|
|
break;
|
|
case '--device':
|
|
cfg.device = readValue(argv, i++, a);
|
|
break;
|
|
case '--serial':
|
|
cfg.serial = readValue(argv, i++, a);
|
|
break;
|
|
default:
|
|
throw new Error(`Unknown flag: ${a}`);
|
|
}
|
|
}
|
|
return cfg;
|
|
}
|