Files
Jakub Piasecki 551c781fb0 feat(flow): support running end-to-end flows without agent (#429)
# Flow E2E

This PR turns flows from replayable tool-call recordings into a
declarative, portable e2e test system that runs headlessly via `argent
flow run <name>` — no LLM in the loop, non-zero exit on failure,
suitable for CI.

Agents can still replay flows via `flow-execute` and get the full output
from each step after it finishes.

## Flow format

Flows are YAML files in `.argent/flows/`, now built from **directives**
interpreted by the runner rather than raw tool calls:

- `launch` — terminate + relaunch the app under test (per-platform app
id map supported); its presence marks a flow as **e2e** (controls its
own start state) vs. a **fragment** (runs against current state,
declares an `executionPrerequisite`, composable via `run:` with
cycle/depth guards).
- `tap`, `type` (focus handshake + optional submit), `scroll-to`
(momentum-free, increments by half the container, `within:` for nested
scrollers), `await`/`assert` (`visible`/`exists`/`hidden`/`text`
conditions), `wait`, `snapshot`, `echo`, and raw `tool:` as escape hatch
— still fully supported, so previously recorded flows keep working
unchanged.
- **Selectors** are `{ text, identifier, role }`; a bare string is a
loose selector resolving identifier-first, then visible text. Any
directive failure hard-stops the flow; later steps report `skip`.

## Execution engine

- Flows store no device id — the runner binds a device at replay
(explicit `--device`/`--platform` or the single booted one).
- Selector resolution runs against **platform-unified UI trees**: iOS
native view hierarchy (full testID coverage, gated on native-devtools
connecting), Android full accessibility hierarchy (full resource-id
coverage), Chromium CDP DOM, and the Vega automation toolkit. A shared
flatten/text-hoisting pass lets `text` assertions read what an
identified container visibly shows. Vega is remote-driven, so touch
directives fail with guidance to use `tv-remote`/`keyboard`.
- Steps wait for the tree to **settle** (two consecutive identical
reads) before acting; `launch` additionally gates on platform
tree-source readiness instead of silently degrading.

## Visual snapshots

`snapshot` diffs a full-resolution screenshot against baselines in
`.argent/flows/__baselines__/<flow>/`, keyed by platform + resolution,
with configurable `maxMismatch` and `--update-baselines`. The status bar
is pinned for the whole run (iOS `simctl status_bar`, Android demo mode)
so clocks/battery don't drive diffs. Baseline/current/diff images are
returned as artifacts on failure (and the written baseline on first
capture/update); a clean pass carries none, keeping reports and agent
context lean.

## Recording

The record-live workflow (`flow-start-recording` → `flow-add-step` →
`flow-finish-recording`) now emits directives automatically: coordinate
taps are rewritten to portable selector `tap:` steps (selector captured
from the pre-tap tree), a leading `restart-app` becomes `launch:`,
`flow-execute` of a sibling fragment becomes `run:`, and device ids are
stripped. The skill instructs the agent to polish the saved YAML
afterwards (fold tap+keyboard into `type:`, swipes-to-reach into
`scroll-to:`, awaits into sugar form).

## Also included

- `argent flow run` / `argent flow list` CLI with per-step report
rendering, artifact materialization, and `--json` output.
- New structured failure codes for flow validation, device resolution,
composition, and devtools readiness.
- Extensive unit test coverage (tree adapters, selector matching,
scrolling, composition, type focus, snapshots, status bar).
2026-07-13 17:15:01 +02:00

69 lines
2.7 KiB
TypeScript

import { beforeEach, describe, expect, it, vi } from "vitest";
// Capture the touch-event train the tool sends to the simulator-server. The
// momentum-free (`settle`) behavior lives entirely in this event sequence, so
// asserting it is how we lock in "no fling".
interface TouchCmd {
cmd: string;
type: "Down" | "Move" | "Up";
x: number;
y: number;
}
const sent: TouchCmd[] = [];
vi.mock("../../src/utils/simulator-client", () => ({
sendCommand: (_api: unknown, cmd: TouchCmd) => {
sent.push(cmd);
},
}));
import { gestureSwipeTool } from "../../src/tools/gesture-swipe";
const services = { simulatorServer: {} } as never;
const base = { udid: "X", fromX: 0.5, fromY: 0.7, toX: 0.5, toY: 0.2 };
/** Length of the trailing run of stationary Moves at the end point before Up. */
function trailingStationaryMoves(events: TouchCmd[], x: number, y: number): number {
let count = 0;
for (let i = events.length - 2; i >= 0; i--) {
const e = events[i];
if (e.type === "Move" && e.x === x && e.y === y) count++;
else break;
}
return count;
}
beforeEach(() => {
sent.length = 0;
});
describe("gesture-swipe", () => {
it("ends with a single Up and no stationary hold when not settling", async () => {
await gestureSwipeTool.execute(services, { ...base, durationMs: 160 });
expect(sent[0]).toMatchObject({ type: "Down", x: 0.5, y: 0.7 });
expect(sent.filter((e) => e.type === "Up")).toHaveLength(1);
expect(sent.at(-1)).toMatchObject({ type: "Up", x: 0.5, y: 0.2 });
// Only the single final interpolation keyframe lands exactly on the end point.
expect(trailingStationaryMoves(sent, 0.5, 0.2)).toBeLessThanOrEqual(1);
});
it("decelerates into the end point (ease-out) before lifting when settling", async () => {
await gestureSwipeTool.execute(services, { ...base, durationMs: 160, settle: true });
// Exactly one lift, at the end point.
expect(sent.filter((e) => e.type === "Up")).toHaveLength(1);
expect(sent.at(-1)).toMatchObject({ type: "Up", x: 0.5, y: 0.2 });
// The momentum-free landing comes from a decelerating trajectory, not a
// stationary hold (which UIKit coalesces away, so the fling survives).
expect(trailingStationaryMoves(sent, 0.5, 0.2)).toBeLessThanOrEqual(1);
// Ease-out: consecutive-sample travel shrinks toward the lift, so the release
// velocity decays to ~0. The last step is a small fraction of the first.
const ys = sent.map((e) => e.y);
const gaps = ys.slice(1).map((y, i) => Math.abs(y - ys[i]));
expect(gaps.at(-1)!).toBeLessThan(gaps[0]);
// Monotonic and in-bounds: every sample sits between the start and end point.
expect(ys.every((y) => y >= 0.2 - 1e-9 && y <= 0.7 + 1e-9)).toBe(true);
});
});