Files
Jakub Piasecki 551c781fb0 feat(flow): support running end-to-end flows without agent (#429)
# Flow E2E

This PR turns flows from replayable tool-call recordings into a
declarative, portable e2e test system that runs headlessly via `argent
flow run <name>` — no LLM in the loop, non-zero exit on failure,
suitable for CI.

Agents can still replay flows via `flow-execute` and get the full output
from each step after it finishes.

## Flow format

Flows are YAML files in `.argent/flows/`, now built from **directives**
interpreted by the runner rather than raw tool calls:

- `launch` — terminate + relaunch the app under test (per-platform app
id map supported); its presence marks a flow as **e2e** (controls its
own start state) vs. a **fragment** (runs against current state,
declares an `executionPrerequisite`, composable via `run:` with
cycle/depth guards).
- `tap`, `type` (focus handshake + optional submit), `scroll-to`
(momentum-free, increments by half the container, `within:` for nested
scrollers), `await`/`assert` (`visible`/`exists`/`hidden`/`text`
conditions), `wait`, `snapshot`, `echo`, and raw `tool:` as escape hatch
— still fully supported, so previously recorded flows keep working
unchanged.
- **Selectors** are `{ text, identifier, role }`; a bare string is a
loose selector resolving identifier-first, then visible text. Any
directive failure hard-stops the flow; later steps report `skip`.

## Execution engine

- Flows store no device id — the runner binds a device at replay
(explicit `--device`/`--platform` or the single booted one).
- Selector resolution runs against **platform-unified UI trees**: iOS
native view hierarchy (full testID coverage, gated on native-devtools
connecting), Android full accessibility hierarchy (full resource-id
coverage), Chromium CDP DOM, and the Vega automation toolkit. A shared
flatten/text-hoisting pass lets `text` assertions read what an
identified container visibly shows. Vega is remote-driven, so touch
directives fail with guidance to use `tv-remote`/`keyboard`.
- Steps wait for the tree to **settle** (two consecutive identical
reads) before acting; `launch` additionally gates on platform
tree-source readiness instead of silently degrading.

## Visual snapshots

`snapshot` diffs a full-resolution screenshot against baselines in
`.argent/flows/__baselines__/<flow>/`, keyed by platform + resolution,
with configurable `maxMismatch` and `--update-baselines`. The status bar
is pinned for the whole run (iOS `simctl status_bar`, Android demo mode)
so clocks/battery don't drive diffs. Baseline/current/diff images are
returned as artifacts on failure (and the written baseline on first
capture/update); a clean pass carries none, keeping reports and agent
context lean.

## Recording

The record-live workflow (`flow-start-recording` → `flow-add-step` →
`flow-finish-recording`) now emits directives automatically: coordinate
taps are rewritten to portable selector `tap:` steps (selector captured
from the pre-tap tree), a leading `restart-app` becomes `launch:`,
`flow-execute` of a sibling fragment becomes `run:`, and device ids are
stripped. The skill instructs the agent to polish the saved YAML
afterwards (fold tap+keyboard into `type:`, swipes-to-reach into
`scroll-to:`, awaits into sugar form).

## Also included

- `argent flow run` / `argent flow list` CLI with per-step report
rendering, artifact materialization, and `--json` output.
- New structured failure codes for flow validation, device resolution,
composition, and devtools readiness.
- Extensive unit test coverage (tree adapters, selector matching,
scrolling, composition, type focus, snapshots, status bar).
2026-07-13 17:15:01 +02:00

145 lines
4.9 KiB
TypeScript

import { describe, it, expect, vi, afterEach } from "vitest";
import supertest from "supertest";
import type { Response } from "superagent";
import { createHttpApp, type HttpAppHandle } from "../src/http";
import type { Registry, InvokeToolOptions } from "@argent/registry";
// Streaming rides the same response path as the update note — pin the checker
// to "no update" so result lines stay minimal and deterministic.
vi.mock("../src/utils/update-checker", () => ({
getUpdateState: vi.fn(() => ({
updateAvailable: false,
updateInstallable: false,
installableVersion: null,
latestVersion: null,
latestPublishedAt: null,
minReleaseAgeMs: 0,
currentVersion: "1.0.0",
})),
isUpdateNoteSuppressed: vi.fn(() => false),
suppressUpdateNote: vi.fn(),
}));
function stubRegistry(
impl: (options: InvokeToolOptions | undefined) => Promise<unknown>
): Registry {
return {
getSnapshot: vi.fn(() => ({ services: new Map(), namespaces: [], tools: ["test-tool"] })),
getTool: vi.fn((name: string) =>
name === "test-tool"
? {
id: "test-tool",
description: "A stub tool for testing",
inputSchema: { type: "object", properties: {} },
services: () => ({}),
execute: async () => ({ ok: true }),
}
: undefined
),
invokeTool: vi.fn((_name: string, _params: unknown, options?: InvokeToolOptions) =>
impl(options)
),
} as unknown as Registry;
}
/** Collect a non-JSON response body as raw text (supertest only parses JSON). */
function collectText(res: Response, cb: (err: Error | null, body: string) => void): void {
let text = "";
res.setEncoding("utf8");
res.on("data", (chunk: string) => (text += chunk));
res.on("end", () => cb(null, text));
}
function parseLines(body: string): Array<Record<string, unknown>> {
return body
.trim()
.split("\n")
.map((line) => JSON.parse(line) as Record<string, unknown>);
}
describe("HTTP NDJSON streaming (Accept: application/x-ndjson)", () => {
let handle: HttpAppHandle;
afterEach(() => {
handle?.dispose();
vi.clearAllMocks();
});
it("streams each emitProgress event as a line, then the terminal result", async () => {
const registry = stubRegistry(async (options) => {
options?.emitProgress?.({ index: 0, status: "pass" });
options?.emitProgress?.({ index: 1, status: "fail" });
return { ok: false, steps: 2 };
});
handle = createHttpApp(registry);
const res = await supertest(handle.app)
.post("/tools/test-tool")
.set("Accept", "application/x-ndjson")
.send({})
.buffer(true)
.parse(collectText)
.expect(200);
expect(res.headers["content-type"]).toContain("application/x-ndjson");
const lines = parseLines(res.body as string);
expect(lines).toEqual([
{ event: "progress", data: { index: 0, status: "pass" } },
{ event: "progress", data: { index: 1, status: "fail" } },
{ event: "result", data: { ok: false, steps: 2 } },
]);
});
it("does NOT pass emitProgress (nor stream) without the Accept header", async () => {
let seenOptions: InvokeToolOptions | undefined;
const registry = stubRegistry(async (options) => {
seenOptions = options;
return { ok: true };
});
handle = createHttpApp(registry);
const res = await supertest(handle.app).post("/tools/test-tool").send({}).expect(200);
expect(seenOptions?.emitProgress).toBeUndefined();
expect(res.headers["content-type"]).toContain("application/json");
expect(res.body).toEqual({ data: { ok: true } });
});
it("delivers a tool failure as an in-band terminal error line", async () => {
const registry = stubRegistry(async (options) => {
options?.emitProgress?.({ index: 0, status: "pass" });
throw new Error("device went away");
});
handle = createHttpApp(registry);
// Headers are already on the wire when the tool fails, so the status stays
// 200 and the error rides the stream's terminal line.
const res = await supertest(handle.app)
.post("/tools/test-tool")
.set("Accept", "application/x-ndjson")
.send({})
.buffer(true)
.parse(collectText)
.expect(200);
const lines = parseLines(res.body as string);
expect(lines).toHaveLength(2);
expect(lines[0]).toEqual({ event: "progress", data: { index: 0, status: "pass" } });
expect(lines[1].event).toBe("error");
expect(String(lines[1].error)).toContain("device went away");
});
it("keeps plain-JSON status codes for failures before the invoke (unknown tool)", async () => {
handle = createHttpApp(stubRegistry(async () => ({ ok: true })));
// The 404 gate fires before the response commits to streaming.
const res = await supertest(handle.app)
.post("/tools/nope")
.set("Accept", "application/x-ndjson")
.send({})
.expect(404);
expect(res.body.error).toContain("nope");
});
});