mirror of
https://github.com/software-mansion/argent.git
synced 2026-09-14 19:27:14 +08:00
551c781fb0
# Flow E2E
This PR turns flows from replayable tool-call recordings into a
declarative, portable e2e test system that runs headlessly via `argent
flow run <name>` — no LLM in the loop, non-zero exit on failure,
suitable for CI.
Agents can still replay flows via `flow-execute` and get the full output
from each step after it finishes.
## Flow format
Flows are YAML files in `.argent/flows/`, now built from **directives**
interpreted by the runner rather than raw tool calls:
- `launch` — terminate + relaunch the app under test (per-platform app
id map supported); its presence marks a flow as **e2e** (controls its
own start state) vs. a **fragment** (runs against current state,
declares an `executionPrerequisite`, composable via `run:` with
cycle/depth guards).
- `tap`, `type` (focus handshake + optional submit), `scroll-to`
(momentum-free, increments by half the container, `within:` for nested
scrollers), `await`/`assert` (`visible`/`exists`/`hidden`/`text`
conditions), `wait`, `snapshot`, `echo`, and raw `tool:` as escape hatch
— still fully supported, so previously recorded flows keep working
unchanged.
- **Selectors** are `{ text, identifier, role }`; a bare string is a
loose selector resolving identifier-first, then visible text. Any
directive failure hard-stops the flow; later steps report `skip`.
## Execution engine
- Flows store no device id — the runner binds a device at replay
(explicit `--device`/`--platform` or the single booted one).
- Selector resolution runs against **platform-unified UI trees**: iOS
native view hierarchy (full testID coverage, gated on native-devtools
connecting), Android full accessibility hierarchy (full resource-id
coverage), Chromium CDP DOM, and the Vega automation toolkit. A shared
flatten/text-hoisting pass lets `text` assertions read what an
identified container visibly shows. Vega is remote-driven, so touch
directives fail with guidance to use `tv-remote`/`keyboard`.
- Steps wait for the tree to **settle** (two consecutive identical
reads) before acting; `launch` additionally gates on platform
tree-source readiness instead of silently degrading.
## Visual snapshots
`snapshot` diffs a full-resolution screenshot against baselines in
`.argent/flows/__baselines__/<flow>/`, keyed by platform + resolution,
with configurable `maxMismatch` and `--update-baselines`. The status bar
is pinned for the whole run (iOS `simctl status_bar`, Android demo mode)
so clocks/battery don't drive diffs. Baseline/current/diff images are
returned as artifacts on failure (and the written baseline on first
capture/update); a clean pass carries none, keeping reports and agent
context lean.
## Recording
The record-live workflow (`flow-start-recording` → `flow-add-step` →
`flow-finish-recording`) now emits directives automatically: coordinate
taps are rewritten to portable selector `tap:` steps (selector captured
from the pre-tap tree), a leading `restart-app` becomes `launch:`,
`flow-execute` of a sibling fragment becomes `run:`, and device ids are
stripped. The skill instructs the agent to polish the saved YAML
afterwards (fold tap+keyboard into `type:`, swipes-to-reach into
`scroll-to:`, awaits into sugar form).
## Also included
- `argent flow run` / `argent flow list` CLI with per-step report
rendering, artifact materialization, and `--json` output.
- New structured failure codes for flow validation, device resolution,
composition, and devtools readiness.
- Extensive unit test coverage (tree adapters, selector matching,
scrolling, composition, type focus, snapshots, status bar).
126 lines
5.5 KiB
TypeScript
126 lines
5.5 KiB
TypeScript
import { describe, it, expect, vi } from "vitest";
|
|
import type { DeviceInfo } from "@argent/registry";
|
|
|
|
// The native-profiler and native-devtools blueprints both open real OS
|
|
// resources (sockets, processes) if we let them reach past the gate. Stub the
|
|
// heavy bits so the only behavior under test is the iOS/Android classification
|
|
// throw.
|
|
vi.mock("@argent/native-devtools-ios", () => ({
|
|
bootstrapDylibPath: () => "/fake/bootstrap.dylib",
|
|
simulatorServerBinaryPath: () => "/fake/sim-server",
|
|
simulatorServerBinaryDir: () => "/fake",
|
|
}));
|
|
|
|
// The proof-of-gate iOS test below runs the native-devtools factory for real,
|
|
// and its ensureEnv shells out to `xcrun simctl` (list / launchctl getenv +
|
|
// setenv / defaults write) against a UDID no simulator has — several real
|
|
// subprocesses that take seconds under the parallel suite load and trip the 5s
|
|
// per-test timeout. Nothing here asserts on the env setup, only on which gate
|
|
// error (if any) a factory throws — so intercept execFile and answer every
|
|
// probe instantly with an empty success.
|
|
vi.mock("node:child_process", async () => {
|
|
const actual = await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
|
return {
|
|
...actual,
|
|
execFile: (
|
|
_cmd: string,
|
|
_args: readonly string[],
|
|
opts: unknown,
|
|
cb?: (err: Error | null, out: { stdout: string; stderr: string }) => void
|
|
) => {
|
|
const callback = (typeof opts === "function" ? opts : cb!) as (
|
|
err: Error | null,
|
|
out: { stdout: string; stderr: string }
|
|
) => void;
|
|
callback(null, { stdout: "", stderr: "" });
|
|
},
|
|
};
|
|
});
|
|
|
|
import { nativeDevtoolsBlueprint } from "../src/blueprints/native-devtools";
|
|
import { nativeProfilerSessionBlueprint } from "../src/blueprints/native-profiler-session";
|
|
import { axServiceBlueprint } from "../src/blueprints/ax-service";
|
|
|
|
function iosDevice(udid: string): DeviceInfo {
|
|
return { id: udid, platform: "ios", kind: "simulator" };
|
|
}
|
|
|
|
function androidDevice(serial: string): DeviceInfo {
|
|
return { id: serial, platform: "android", kind: "emulator" };
|
|
}
|
|
|
|
describe("iOS-only blueprints reject Android targets up-front", () => {
|
|
// Agents see both iOS and Android targets in list-devices. Feeding an Android
|
|
// serial to a tool backed by an iOS-only blueprint (native-devtools,
|
|
// native-profiler-session) used to resolve the service, fail deep in
|
|
// launchctl / xctrace / socket connect, and surface as an opaque error.
|
|
// These gates turn that into a clear "iOS-only, pick an iOS udid" message
|
|
// at the blueprint boundary — using the caller-supplied DeviceInfo, not a
|
|
// re-classification of the URN payload.
|
|
|
|
it("native-devtools blueprint rejects an Android device with a targeted error", async () => {
|
|
const device = androidDevice("emulator-5554");
|
|
await expect(nativeDevtoolsBlueprint.factory({}, device, { device })).rejects.toThrow(
|
|
/NativeDevtools is iOS-only.*classifies as android/
|
|
);
|
|
});
|
|
|
|
it("native-profiler-session blueprint accepts an Android device (Perfetto backend)", async () => {
|
|
const device = androidDevice("emulator-5556");
|
|
const instance = await nativeProfilerSessionBlueprint.factory({}, device, { device });
|
|
expect(instance.api.platform).toBe("android");
|
|
await instance.dispose();
|
|
});
|
|
|
|
it("native-devtools blueprint does NOT gate an iOS DeviceInfo (gate is one-sided)", async () => {
|
|
// Proof-of-gate: if the caller hands us an iOS device we pass the
|
|
// `device.platform !== "ios"` check. Whether the rest of the factory
|
|
// resolves or rejects depends on socket state which this test doesn't
|
|
// control — the invariant we care about is that the failure mode is
|
|
// never the iOS-only gate message for an iOS target.
|
|
const device = iosDevice("11111111-2222-3333-4444-555555555555");
|
|
let threwGateError = false;
|
|
try {
|
|
const instance = await nativeDevtoolsBlueprint.factory({}, device, { device });
|
|
// If the factory resolves, dispose it so we don't leak the socket watcher.
|
|
await instance.dispose();
|
|
} catch (e) {
|
|
if (e instanceof Error && /NativeDevtools is iOS-only/.test(e.message)) {
|
|
threwGateError = true;
|
|
}
|
|
}
|
|
expect(threwGateError).toBe(false);
|
|
});
|
|
|
|
it("native-devtools blueprint rejects when caller forgets options.device", async () => {
|
|
// Defensive: without a DeviceInfo the factory has no way to gate on
|
|
// platform — surface a clear error pointing at the helper rather than
|
|
// silently defaulting.
|
|
const stub = iosDevice("ignored");
|
|
await expect(nativeDevtoolsBlueprint.factory({}, stub)).rejects.toThrow(
|
|
/requires a resolved DeviceInfo via options\.device/
|
|
);
|
|
});
|
|
|
|
it("native-profiler-session blueprint rejects when caller forgets options.device", async () => {
|
|
const stub = iosDevice("ignored");
|
|
await expect(nativeProfilerSessionBlueprint.factory({}, stub)).rejects.toThrow(
|
|
/requires a resolved DeviceInfo via options\.device/
|
|
);
|
|
});
|
|
|
|
it("ax-service blueprint rejects an Android device with a targeted error", async () => {
|
|
const device = androidDevice("emulator-5554");
|
|
await expect(axServiceBlueprint.factory({}, device, { device })).rejects.toThrow(
|
|
/AXService is iOS-only.*classifies as android.*uiautomator/
|
|
);
|
|
});
|
|
|
|
it("ax-service blueprint rejects when caller forgets options.device", async () => {
|
|
const stub = iosDevice("ignored");
|
|
await expect(axServiceBlueprint.factory({}, stub)).rejects.toThrow(
|
|
/requires a resolved DeviceInfo via options\.device/
|
|
);
|
|
});
|
|
});
|