mirror of
https://github.com/software-mansion/argent.git
synced 2026-09-14 19:27:14 +08:00
551c781fb0
# Flow E2E
This PR turns flows from replayable tool-call recordings into a
declarative, portable e2e test system that runs headlessly via `argent
flow run <name>` — no LLM in the loop, non-zero exit on failure,
suitable for CI.
Agents can still replay flows via `flow-execute` and get the full output
from each step after it finishes.
## Flow format
Flows are YAML files in `.argent/flows/`, now built from **directives**
interpreted by the runner rather than raw tool calls:
- `launch` — terminate + relaunch the app under test (per-platform app
id map supported); its presence marks a flow as **e2e** (controls its
own start state) vs. a **fragment** (runs against current state,
declares an `executionPrerequisite`, composable via `run:` with
cycle/depth guards).
- `tap`, `type` (focus handshake + optional submit), `scroll-to`
(momentum-free, increments by half the container, `within:` for nested
scrollers), `await`/`assert` (`visible`/`exists`/`hidden`/`text`
conditions), `wait`, `snapshot`, `echo`, and raw `tool:` as escape hatch
— still fully supported, so previously recorded flows keep working
unchanged.
- **Selectors** are `{ text, identifier, role }`; a bare string is a
loose selector resolving identifier-first, then visible text. Any
directive failure hard-stops the flow; later steps report `skip`.
## Execution engine
- Flows store no device id — the runner binds a device at replay
(explicit `--device`/`--platform` or the single booted one).
- Selector resolution runs against **platform-unified UI trees**: iOS
native view hierarchy (full testID coverage, gated on native-devtools
connecting), Android full accessibility hierarchy (full resource-id
coverage), Chromium CDP DOM, and the Vega automation toolkit. A shared
flatten/text-hoisting pass lets `text` assertions read what an
identified container visibly shows. Vega is remote-driven, so touch
directives fail with guidance to use `tv-remote`/`keyboard`.
- Steps wait for the tree to **settle** (two consecutive identical
reads) before acting; `launch` additionally gates on platform
tree-source readiness instead of silently degrading.
## Visual snapshots
`snapshot` diffs a full-resolution screenshot against baselines in
`.argent/flows/__baselines__/<flow>/`, keyed by platform + resolution,
with configurable `maxMismatch` and `--update-baselines`. The status bar
is pinned for the whole run (iOS `simctl status_bar`, Android demo mode)
so clocks/battery don't drive diffs. Baseline/current/diff images are
returned as artifacts on failure (and the written baseline on first
capture/update); a clean pass carries none, keeping reports and agent
context lean.
## Recording
The record-live workflow (`flow-start-recording` → `flow-add-step` →
`flow-finish-recording`) now emits directives automatically: coordinate
taps are rewritten to portable selector `tap:` steps (selector captured
from the pre-tap tree), a leading `restart-app` becomes `launch:`,
`flow-execute` of a sibling fragment becomes `run:`, and device ids are
stripped. The skill instructs the agent to polish the saved YAML
afterwards (fold tap+keyboard into `type:`, swipes-to-reach into
`scroll-to:`, awaits into sugar form).
## Also included
- `argent flow run` / `argent flow list` CLI with per-step report
rendering, artifact materialization, and `--json` output.
- New structured failure codes for flow validation, device resolution,
composition, and devtools readiness.
- Extensive unit test coverage (tree adapters, selector matching,
scrolling, composition, type focus, snapshots, status bar).
118 lines
4.8 KiB
TypeScript
118 lines
4.8 KiB
TypeScript
import { describe, it, expect, vi } from "vitest";
|
|
import { gestureScrollTool } from "../src/tools/gesture-scroll";
|
|
import { gestureSwipeTool } from "../src/tools/gesture-swipe";
|
|
import { assertSupported, UnsupportedOperationError } from "../src/utils/capability";
|
|
import { resolveDevice } from "../src/utils/device-info";
|
|
|
|
// The scroll/swipe split: a desktop renderer scrolls with wheel events
|
|
// (gesture-scroll, chromium-only) while touch platforms scroll with a drag
|
|
// gesture (gesture-swipe, ios/android-only). These tests pin both the wheel
|
|
// dispatch math and the capability fence between the two tools.
|
|
|
|
const chromiumDevice = resolveDevice("chromium-cdp-19222");
|
|
const iosDevice = resolveDevice("AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA");
|
|
const androidDevice = resolveDevice("emulator-5554");
|
|
|
|
function fakeChromiumApi(visibility = "visible") {
|
|
return {
|
|
getViewport: () => ({ width: 800, height: 600, devicePixelRatio: 2 }),
|
|
server: { sendWheel: vi.fn().mockResolvedValue(undefined) },
|
|
cdp: { send: vi.fn().mockResolvedValue({ result: { value: visibility } }) },
|
|
};
|
|
}
|
|
|
|
describe("gesture-scroll", () => {
|
|
it("dispatches chunked wheel deltas at the anchor point, totalling the requested fraction", async () => {
|
|
const api = fakeChromiumApi();
|
|
const result = await gestureScrollTool.execute(
|
|
{ chromium: api } as never,
|
|
{ udid: "chromium-cdp-19222", x: 0.5, y: 0.65, deltaY: 0.25, durationMs: 64 } as never
|
|
);
|
|
expect(result.scrolled).toBe(true);
|
|
const calls = api.server.sendWheel.mock.calls;
|
|
expect(calls.length).toBeGreaterThan(1);
|
|
expect(calls[0]![0]).toEqual({ x: 0.5, y: 0.65 });
|
|
const totalDx = calls.reduce((sum, c) => sum + (c[1] as number), 0);
|
|
const totalDy = calls.reduce((sum, c) => sum + (c[2] as number), 0);
|
|
expect(totalDx).toBeCloseTo(0, 5);
|
|
expect(totalDy).toBeCloseTo(0.25 * 600, 5);
|
|
});
|
|
|
|
it("supports horizontal and negative (scroll-back-up) deltas", async () => {
|
|
const api = fakeChromiumApi();
|
|
await gestureScrollTool.execute(
|
|
{ chromium: api } as never,
|
|
{
|
|
udid: "chromium-cdp-19222",
|
|
x: 0.5,
|
|
y: 0.5,
|
|
deltaX: 0.1,
|
|
deltaY: -0.5,
|
|
durationMs: 32,
|
|
} as never
|
|
);
|
|
const calls = api.server.sendWheel.mock.calls;
|
|
const totalDx = calls.reduce((sum, c) => sum + (c[1] as number), 0);
|
|
const totalDy = calls.reduce((sum, c) => sum + (c[2] as number), 0);
|
|
expect(totalDx).toBeCloseTo(0.1 * 800, 5);
|
|
expect(totalDy).toBeCloseTo(-0.5 * 600, 5);
|
|
});
|
|
|
|
it("fails fast with an actionable error when the window is hidden", async () => {
|
|
// A hidden window halts the input pipeline: wheel dispatches would stall
|
|
// past the CDP timeout, so the tool must refuse before dispatching any.
|
|
const api = fakeChromiumApi("hidden");
|
|
await expect(
|
|
gestureScrollTool.execute(
|
|
{ chromium: api } as never,
|
|
{ udid: "chromium-cdp-19222", x: 0.5, y: 0.5, deltaY: 0.25 } as never
|
|
)
|
|
).rejects.toThrow(/hidden/);
|
|
expect(api.server.sendWheel).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("proceeds with the scroll when the visibility probe itself rejects", async () => {
|
|
// A failed read proves nothing: a Runtime.evaluate rejected mid-navigation
|
|
// (execution context destroyed) must not fail the scroll — the wheel
|
|
// dispatch needs no JS context. Only an explicit "hidden" refuses.
|
|
const api = fakeChromiumApi();
|
|
api.cdp.send = vi.fn().mockRejectedValue(new Error("Execution context was destroyed"));
|
|
const result = await gestureScrollTool.execute(
|
|
{ chromium: api } as never,
|
|
{ udid: "chromium-cdp-19222", x: 0.5, y: 0.5, deltaY: 0.25, durationMs: 32 } as never
|
|
);
|
|
expect(result.scrolled).toBe(true);
|
|
expect(api.server.sendWheel).toHaveBeenCalled();
|
|
});
|
|
|
|
it("schema rejects a scroll with no delta", () => {
|
|
const parsed = gestureScrollTool.zodSchema!.safeParse({
|
|
udid: "chromium-cdp-19222",
|
|
x: 0.5,
|
|
y: 0.5,
|
|
});
|
|
expect(parsed.success).toBe(false);
|
|
});
|
|
|
|
it("is chromium-only: capability gate rejects iOS and Android targets", () => {
|
|
expect(() =>
|
|
assertSupported("gesture-scroll", gestureScrollTool.capability!, chromiumDevice)
|
|
).not.toThrow();
|
|
expect(() =>
|
|
assertSupported("gesture-scroll", gestureScrollTool.capability!, iosDevice)
|
|
).toThrow(UnsupportedOperationError);
|
|
expect(() =>
|
|
assertSupported("gesture-scroll", gestureScrollTool.capability!, androidDevice)
|
|
).toThrow(UnsupportedOperationError);
|
|
});
|
|
});
|
|
|
|
describe("gesture-swipe chromium lockout", () => {
|
|
it("no longer declares chromium support — the gate names the tool clearly", () => {
|
|
expect(gestureSwipeTool.capability).not.toHaveProperty("chromium");
|
|
expect(() =>
|
|
assertSupported("gesture-swipe", gestureSwipeTool.capability!, chromiumDevice)
|
|
).toThrow(/gesture-swipe.*not supported on chromium/);
|
|
});
|
|
});
|