Files
Jakub Piasecki 551c781fb0 feat(flow): support running end-to-end flows without agent (#429)
# Flow E2E

This PR turns flows from replayable tool-call recordings into a
declarative, portable e2e test system that runs headlessly via `argent
flow run <name>` — no LLM in the loop, non-zero exit on failure,
suitable for CI.

Agents can still replay flows via `flow-execute` and get the full output
from each step after it finishes.

## Flow format

Flows are YAML files in `.argent/flows/`, now built from **directives**
interpreted by the runner rather than raw tool calls:

- `launch` — terminate + relaunch the app under test (per-platform app
id map supported); its presence marks a flow as **e2e** (controls its
own start state) vs. a **fragment** (runs against current state,
declares an `executionPrerequisite`, composable via `run:` with
cycle/depth guards).
- `tap`, `type` (focus handshake + optional submit), `scroll-to`
(momentum-free, increments by half the container, `within:` for nested
scrollers), `await`/`assert` (`visible`/`exists`/`hidden`/`text`
conditions), `wait`, `snapshot`, `echo`, and raw `tool:` as escape hatch
— still fully supported, so previously recorded flows keep working
unchanged.
- **Selectors** are `{ text, identifier, role }`; a bare string is a
loose selector resolving identifier-first, then visible text. Any
directive failure hard-stops the flow; later steps report `skip`.

## Execution engine

- Flows store no device id — the runner binds a device at replay
(explicit `--device`/`--platform` or the single booted one).
- Selector resolution runs against **platform-unified UI trees**: iOS
native view hierarchy (full testID coverage, gated on native-devtools
connecting), Android full accessibility hierarchy (full resource-id
coverage), Chromium CDP DOM, and the Vega automation toolkit. A shared
flatten/text-hoisting pass lets `text` assertions read what an
identified container visibly shows. Vega is remote-driven, so touch
directives fail with guidance to use `tv-remote`/`keyboard`.
- Steps wait for the tree to **settle** (two consecutive identical
reads) before acting; `launch` additionally gates on platform
tree-source readiness instead of silently degrading.

## Visual snapshots

`snapshot` diffs a full-resolution screenshot against baselines in
`.argent/flows/__baselines__/<flow>/`, keyed by platform + resolution,
with configurable `maxMismatch` and `--update-baselines`. The status bar
is pinned for the whole run (iOS `simctl status_bar`, Android demo mode)
so clocks/battery don't drive diffs. Baseline/current/diff images are
returned as artifacts on failure (and the written baseline on first
capture/update); a clean pass carries none, keeping reports and agent
context lean.

## Recording

The record-live workflow (`flow-start-recording` → `flow-add-step` →
`flow-finish-recording`) now emits directives automatically: coordinate
taps are rewritten to portable selector `tap:` steps (selector captured
from the pre-tap tree), a leading `restart-app` becomes `launch:`,
`flow-execute` of a sibling fragment becomes `run:`, and device ids are
stripped. The skill instructs the agent to polish the saved YAML
afterwards (fold tap+keyboard into `type:`, swipes-to-reach into
`scroll-to:`, awaits into sugar form).

## Also included

- `argent flow run` / `argent flow list` CLI with per-step report
rendering, artifact materialization, and `--json` output.
- New structured failure codes for flow validation, device resolution,
composition, and devtools readiness.
- Extensive unit test coverage (tree adapters, selector matching,
scrolling, composition, type focus, snapshots, status bar).
2026-07-13 17:15:01 +02:00

115 lines
4.5 KiB
TypeScript

import { describe, it, expect, vi, beforeEach } from "vitest";
import type { DeviceInfo } from "@argent/registry";
const execFileMock = vi.fn();
vi.mock("node:child_process", async () => {
const actual = await vi.importActual<typeof import("node:child_process")>("node:child_process");
return {
...actual,
execFile: (
cmd: string,
args: readonly string[],
opts: unknown,
cb?: (err: Error | null, out: { stdout: string; stderr: string }) => void
) => {
const callback = typeof opts === "function" ? opts : cb!;
const options = typeof opts === "function" ? undefined : opts;
const result = execFileMock(cmd, args, options);
if (result instanceof Error) callback(result, { stdout: "", stderr: "" });
else callback(null, result ?? { stdout: "", stderr: "" });
},
};
});
vi.mock("../src/utils/android-binary", () => ({
resolveAndroidBinary: vi.fn(async (name: "adb" | "emulator") => name),
__resetAndroidBinaryCacheForTesting: () => {},
}));
// The suite-wide setup file (test/setup/stub-status-bar.ts) replaces this module
// so other tests never shell out; this file is the one place that tests the real
// implementation (against the execFile mock above), so opt back in.
vi.unmock("../src/utils/status-bar");
import { pinStatusBar, restoreStatusBar } from "../src/utils/status-bar";
const ANDROID_DEVICE: DeviceInfo = {
id: "emulator-5554",
platform: "android",
kind: "emulator",
};
/** Shell payloads of every `adb -s <serial> shell <cmd>` call, in order. */
function shellCalls(): string[] {
return execFileMock.mock.calls
.filter(([cmd, args]) => cmd === "adb" && args[0] === "-s" && args[2] === "shell")
.map(([, args]) => args[3] as string);
}
beforeEach(() => {
execFileMock.mockReset();
});
describe("pinStatusBar (android)", () => {
it("returns true when every demo-mode command succeeds", async () => {
execFileMock.mockReturnValue({ stdout: "", stderr: "" });
expect(await pinStatusBar(ANDROID_DEVICE)).toBe(true);
expect(shellCalls().some((c) => c.includes("command exit"))).toBe(false);
});
it("sends the demo-mode exit broadcast when a command fails after enter", async () => {
// Fail the clock broadcast — demo mode has already been entered by then,
// so pinStatusBar must undo it rather than leave the device pinned with
// no restore scheduled (the caller skips restoreStatusBar on false).
execFileMock.mockImplementation((cmd: string, args: string[]) => {
const shell = args[3] ?? "";
if (shell.includes("command clock")) return new Error("adb: device offline");
return { stdout: "", stderr: "" };
});
expect(await pinStatusBar(ANDROID_DEVICE)).toBe(false);
expect(shellCalls().some((c) => c.includes("command exit"))).toBe(true);
expect(shellCalls().some((c) => c.includes("sysui_demo_allowed 0"))).toBe(true);
});
it("reports pinned when the cleanup exit broadcast fails too, so the run-end restore fires", async () => {
// Demo mode was entered and could not be exited (transient adb). Returning
// false here would tell the caller nothing needs restoring, leaving the
// frozen clock/battery applied after the run — so report `true` instead
// and let the caller's teardown restore retry.
execFileMock.mockImplementation((cmd: string, args: string[]) => {
const shell = args[3] ?? "";
if (shell.includes("command clock") || shell.includes("command exit"))
return new Error("adb: device offline");
return { stdout: "", stderr: "" };
});
await expect(pinStatusBar(ANDROID_DEVICE)).resolves.toBe(true);
});
});
describe("restoreStatusBar (android)", () => {
it("exits demo mode and resets sysui_demo_allowed", async () => {
execFileMock.mockReturnValue({ stdout: "", stderr: "" });
expect(await restoreStatusBar(ANDROID_DEVICE)).toBe(true);
expect(shellCalls()).toEqual([
"am broadcast -a com.android.systemui.demo -e command exit",
"settings put global sysui_demo_allowed 0",
]);
});
it("still resets sysui_demo_allowed when the exit broadcast fails, and reports failure", async () => {
execFileMock.mockImplementation((cmd: string, args: string[]) => {
const shell = args[3] ?? "";
if (shell.includes("command exit")) return new Error("adb: device offline");
return { stdout: "", stderr: "" };
});
expect(await restoreStatusBar(ANDROID_DEVICE)).toBe(false);
expect(shellCalls().some((c) => c.includes("sysui_demo_allowed 0"))).toBe(true);
});
});