Files
Jakub Piasecki 551c781fb0 feat(flow): support running end-to-end flows without agent (#429)
# Flow E2E

This PR turns flows from replayable tool-call recordings into a
declarative, portable e2e test system that runs headlessly via `argent
flow run <name>` — no LLM in the loop, non-zero exit on failure,
suitable for CI.

Agents can still replay flows via `flow-execute` and get the full output
from each step after it finishes.

## Flow format

Flows are YAML files in `.argent/flows/`, now built from **directives**
interpreted by the runner rather than raw tool calls:

- `launch` — terminate + relaunch the app under test (per-platform app
id map supported); its presence marks a flow as **e2e** (controls its
own start state) vs. a **fragment** (runs against current state,
declares an `executionPrerequisite`, composable via `run:` with
cycle/depth guards).
- `tap`, `type` (focus handshake + optional submit), `scroll-to`
(momentum-free, increments by half the container, `within:` for nested
scrollers), `await`/`assert` (`visible`/`exists`/`hidden`/`text`
conditions), `wait`, `snapshot`, `echo`, and raw `tool:` as escape hatch
— still fully supported, so previously recorded flows keep working
unchanged.
- **Selectors** are `{ text, identifier, role }`; a bare string is a
loose selector resolving identifier-first, then visible text. Any
directive failure hard-stops the flow; later steps report `skip`.

## Execution engine

- Flows store no device id — the runner binds a device at replay
(explicit `--device`/`--platform` or the single booted one).
- Selector resolution runs against **platform-unified UI trees**: iOS
native view hierarchy (full testID coverage, gated on native-devtools
connecting), Android full accessibility hierarchy (full resource-id
coverage), Chromium CDP DOM, and the Vega automation toolkit. A shared
flatten/text-hoisting pass lets `text` assertions read what an
identified container visibly shows. Vega is remote-driven, so touch
directives fail with guidance to use `tv-remote`/`keyboard`.
- Steps wait for the tree to **settle** (two consecutive identical
reads) before acting; `launch` additionally gates on platform
tree-source readiness instead of silently degrading.

## Visual snapshots

`snapshot` diffs a full-resolution screenshot against baselines in
`.argent/flows/__baselines__/<flow>/`, keyed by platform + resolution,
with configurable `maxMismatch` and `--update-baselines`. The status bar
is pinned for the whole run (iOS `simctl status_bar`, Android demo mode)
so clocks/battery don't drive diffs. Baseline/current/diff images are
returned as artifacts on failure (and the written baseline on first
capture/update); a clean pass carries none, keeping reports and agent
context lean.

## Recording

The record-live workflow (`flow-start-recording` → `flow-add-step` →
`flow-finish-recording`) now emits directives automatically: coordinate
taps are rewritten to portable selector `tap:` steps (selector captured
from the pre-tap tree), a leading `restart-app` becomes `launch:`,
`flow-execute` of a sibling fragment becomes `run:`, and device ids are
stripped. The skill instructs the agent to polish the saved YAML
afterwards (fold tap+keyboard into `type:`, swipes-to-reach into
`scroll-to:`, awaits into sugar form).

## Also included

- `argent flow run` / `argent flow list` CLI with per-step report
rendering, artifact materialization, and `--json` output.
- New structured failure codes for flow validation, device resolution,
composition, and devtools readiness.
- Extensive unit test coverage (tree adapters, selector matching,
scrolling, composition, type focus, snapshots, status bar).
2026-07-13 17:15:01 +02:00

119 lines
4.5 KiB
TypeScript

import { describe, it, expect, vi, afterEach } from "vitest";
import { createServer, type Server, type IncomingMessage, type ServerResponse } from "node:http";
import type { AddressInfo } from "node:net";
import { createToolsClient } from "../src/tools-client.js";
let server: Server | undefined;
/**
* Stand up a stub tool-server: GET /tools advertises one tool ("streamy",
* no fileInputs so the file boundary stays out of the way); POST behavior is
* supplied per test. ARGENT_TOOLS_URL routes the client at it.
*/
async function startServer(
onInvoke: (req: IncomingMessage, res: ServerResponse) => void
): Promise<void> {
server = createServer((req, res) => {
if (req.method === "GET" && req.url === "/tools") {
res.setHeader("Content-Type", "application/json");
res.end(JSON.stringify({ tools: [{ name: "streamy", description: "", inputSchema: {} }] }));
return;
}
if (req.method === "POST" && req.url === "/tools/streamy") {
onInvoke(req, res);
return;
}
res.statusCode = 404;
res.end(JSON.stringify({ error: "not found" }));
});
await new Promise<void>((resolve) => server!.listen(0, "127.0.0.1", resolve));
const { port } = server!.address() as AddressInfo;
vi.stubEnv("ARGENT_TOOLS_URL", `http://127.0.0.1:${port}`);
}
afterEach(async () => {
vi.unstubAllEnvs();
if (server) {
await new Promise<void>((resolve) => server!.close(() => resolve()));
server = undefined;
}
});
describe("callTool progress streaming", () => {
it("fires onProgress per NDJSON line and resolves with the terminal result", async () => {
await startServer((_req, res) => {
res.writeHead(200, { "Content-Type": "application/x-ndjson" });
res.write(`${JSON.stringify({ event: "progress", data: { index: 0 } })}\n`);
res.write(`${JSON.stringify({ event: "progress", data: { index: 1 } })}\n`);
res.end(`${JSON.stringify({ event: "result", data: { ok: true }, note: "hi" })}\n`);
});
const events: unknown[] = [];
const { callTool } = createToolsClient();
const result = await callTool("streamy", {}, { onProgress: (e) => events.push(e) });
expect(events).toEqual([{ index: 0 }, { index: 1 }]);
expect(result.data).toEqual({ ok: true });
expect(result.note).toBe("hi");
});
it("sends the Accept header only when a progress consumer is attached", async () => {
const accepts: Array<string | undefined> = [];
await startServer((req, res) => {
accepts.push(req.headers.accept);
res.setHeader("Content-Type", "application/json");
res.end(JSON.stringify({ data: { ok: true } }));
});
const { callTool } = createToolsClient();
await callTool("streamy", {});
await callTool("streamy", {}, { onProgress: () => {} });
expect(accepts[0] ?? "").not.toContain("application/x-ndjson");
expect(accepts[1]).toContain("application/x-ndjson");
});
it("falls back to the buffered path against a pre-streaming server", async () => {
// An old server ignores the Accept header and replies plain JSON.
await startServer((_req, res) => {
res.setHeader("Content-Type", "application/json");
res.end(JSON.stringify({ data: { ok: true } }));
});
const events: unknown[] = [];
const { callTool } = createToolsClient();
const result = await callTool("streamy", {}, { onProgress: (e) => events.push(e) });
expect(events).toEqual([]);
expect(result.data).toEqual({ ok: true });
});
it("rejects with the in-band terminal error", async () => {
await startServer((_req, res) => {
res.writeHead(200, { "Content-Type": "application/x-ndjson" });
res.write(`${JSON.stringify({ event: "progress", data: { index: 0 } })}\n`);
res.end(`${JSON.stringify({ event: "error", error: "kaput" })}\n`);
});
const events: unknown[] = [];
const { callTool } = createToolsClient();
await expect(callTool("streamy", {}, { onProgress: (e) => events.push(e) })).rejects.toThrow(
"kaput"
);
expect(events).toEqual([{ index: 0 }]);
});
it("rejects when the stream ends without a terminal line (connection lost)", async () => {
await startServer((_req, res) => {
res.writeHead(200, { "Content-Type": "application/x-ndjson" });
res.write(`${JSON.stringify({ event: "progress", data: { index: 0 } })}\n`);
res.end();
});
const { callTool } = createToolsClient();
await expect(callTool("streamy", {}, { onProgress: () => {} })).rejects.toThrow(
/without a result/
);
});
});