mirror of
https://github.com/software-mansion/argent.git
synced 2026-09-14 19:27:14 +08:00
551c781fb0
# Flow E2E
This PR turns flows from replayable tool-call recordings into a
declarative, portable e2e test system that runs headlessly via `argent
flow run <name>` — no LLM in the loop, non-zero exit on failure,
suitable for CI.
Agents can still replay flows via `flow-execute` and get the full output
from each step after it finishes.
## Flow format
Flows are YAML files in `.argent/flows/`, now built from **directives**
interpreted by the runner rather than raw tool calls:
- `launch` — terminate + relaunch the app under test (per-platform app
id map supported); its presence marks a flow as **e2e** (controls its
own start state) vs. a **fragment** (runs against current state,
declares an `executionPrerequisite`, composable via `run:` with
cycle/depth guards).
- `tap`, `type` (focus handshake + optional submit), `scroll-to`
(momentum-free, increments by half the container, `within:` for nested
scrollers), `await`/`assert` (`visible`/`exists`/`hidden`/`text`
conditions), `wait`, `snapshot`, `echo`, and raw `tool:` as escape hatch
— still fully supported, so previously recorded flows keep working
unchanged.
- **Selectors** are `{ text, identifier, role }`; a bare string is a
loose selector resolving identifier-first, then visible text. Any
directive failure hard-stops the flow; later steps report `skip`.
## Execution engine
- Flows store no device id — the runner binds a device at replay
(explicit `--device`/`--platform` or the single booted one).
- Selector resolution runs against **platform-unified UI trees**: iOS
native view hierarchy (full testID coverage, gated on native-devtools
connecting), Android full accessibility hierarchy (full resource-id
coverage), Chromium CDP DOM, and the Vega automation toolkit. A shared
flatten/text-hoisting pass lets `text` assertions read what an
identified container visibly shows. Vega is remote-driven, so touch
directives fail with guidance to use `tv-remote`/`keyboard`.
- Steps wait for the tree to **settle** (two consecutive identical
reads) before acting; `launch` additionally gates on platform
tree-source readiness instead of silently degrading.
## Visual snapshots
`snapshot` diffs a full-resolution screenshot against baselines in
`.argent/flows/__baselines__/<flow>/`, keyed by platform + resolution,
with configurable `maxMismatch` and `--update-baselines`. The status bar
is pinned for the whole run (iOS `simctl status_bar`, Android demo mode)
so clocks/battery don't drive diffs. Baseline/current/diff images are
returned as artifacts on failure (and the written baseline on first
capture/update); a clean pass carries none, keeping reports and agent
context lean.
## Recording
The record-live workflow (`flow-start-recording` → `flow-add-step` →
`flow-finish-recording`) now emits directives automatically: coordinate
taps are rewritten to portable selector `tap:` steps (selector captured
from the pre-tap tree), a leading `restart-app` becomes `launch:`,
`flow-execute` of a sibling fragment becomes `run:`, and device ids are
stripped. The skill instructs the agent to polish the saved YAML
afterwards (fold tap+keyboard into `type:`, swipes-to-reach into
`scroll-to:`, awaits into sugar form).
## Also included
- `argent flow run` / `argent flow list` CLI with per-step report
rendering, artifact materialization, and `--json` output.
- New structured failure codes for flow validation, device resolution,
composition, and devtools readiness.
- Extensive unit test coverage (tree adapters, selector matching,
scrolling, composition, type focus, snapshots, status bar).
145 lines
4.9 KiB
TypeScript
145 lines
4.9 KiB
TypeScript
import { describe, it, expect, vi, afterEach } from "vitest";
|
|
import supertest from "supertest";
|
|
import type { Response } from "superagent";
|
|
import { createHttpApp, type HttpAppHandle } from "../src/http";
|
|
import type { Registry, InvokeToolOptions } from "@argent/registry";
|
|
|
|
// Streaming rides the same response path as the update note — pin the checker
|
|
// to "no update" so result lines stay minimal and deterministic.
|
|
vi.mock("../src/utils/update-checker", () => ({
|
|
getUpdateState: vi.fn(() => ({
|
|
updateAvailable: false,
|
|
updateInstallable: false,
|
|
installableVersion: null,
|
|
latestVersion: null,
|
|
latestPublishedAt: null,
|
|
minReleaseAgeMs: 0,
|
|
currentVersion: "1.0.0",
|
|
})),
|
|
isUpdateNoteSuppressed: vi.fn(() => false),
|
|
suppressUpdateNote: vi.fn(),
|
|
}));
|
|
|
|
function stubRegistry(
|
|
impl: (options: InvokeToolOptions | undefined) => Promise<unknown>
|
|
): Registry {
|
|
return {
|
|
getSnapshot: vi.fn(() => ({ services: new Map(), namespaces: [], tools: ["test-tool"] })),
|
|
getTool: vi.fn((name: string) =>
|
|
name === "test-tool"
|
|
? {
|
|
id: "test-tool",
|
|
description: "A stub tool for testing",
|
|
inputSchema: { type: "object", properties: {} },
|
|
services: () => ({}),
|
|
execute: async () => ({ ok: true }),
|
|
}
|
|
: undefined
|
|
),
|
|
invokeTool: vi.fn((_name: string, _params: unknown, options?: InvokeToolOptions) =>
|
|
impl(options)
|
|
),
|
|
} as unknown as Registry;
|
|
}
|
|
|
|
/** Collect a non-JSON response body as raw text (supertest only parses JSON). */
|
|
function collectText(res: Response, cb: (err: Error | null, body: string) => void): void {
|
|
let text = "";
|
|
res.setEncoding("utf8");
|
|
res.on("data", (chunk: string) => (text += chunk));
|
|
res.on("end", () => cb(null, text));
|
|
}
|
|
|
|
function parseLines(body: string): Array<Record<string, unknown>> {
|
|
return body
|
|
.trim()
|
|
.split("\n")
|
|
.map((line) => JSON.parse(line) as Record<string, unknown>);
|
|
}
|
|
|
|
describe("HTTP NDJSON streaming (Accept: application/x-ndjson)", () => {
|
|
let handle: HttpAppHandle;
|
|
|
|
afterEach(() => {
|
|
handle?.dispose();
|
|
vi.clearAllMocks();
|
|
});
|
|
|
|
it("streams each emitProgress event as a line, then the terminal result", async () => {
|
|
const registry = stubRegistry(async (options) => {
|
|
options?.emitProgress?.({ index: 0, status: "pass" });
|
|
options?.emitProgress?.({ index: 1, status: "fail" });
|
|
return { ok: false, steps: 2 };
|
|
});
|
|
handle = createHttpApp(registry);
|
|
|
|
const res = await supertest(handle.app)
|
|
.post("/tools/test-tool")
|
|
.set("Accept", "application/x-ndjson")
|
|
.send({})
|
|
.buffer(true)
|
|
.parse(collectText)
|
|
.expect(200);
|
|
|
|
expect(res.headers["content-type"]).toContain("application/x-ndjson");
|
|
const lines = parseLines(res.body as string);
|
|
expect(lines).toEqual([
|
|
{ event: "progress", data: { index: 0, status: "pass" } },
|
|
{ event: "progress", data: { index: 1, status: "fail" } },
|
|
{ event: "result", data: { ok: false, steps: 2 } },
|
|
]);
|
|
});
|
|
|
|
it("does NOT pass emitProgress (nor stream) without the Accept header", async () => {
|
|
let seenOptions: InvokeToolOptions | undefined;
|
|
const registry = stubRegistry(async (options) => {
|
|
seenOptions = options;
|
|
return { ok: true };
|
|
});
|
|
handle = createHttpApp(registry);
|
|
|
|
const res = await supertest(handle.app).post("/tools/test-tool").send({}).expect(200);
|
|
|
|
expect(seenOptions?.emitProgress).toBeUndefined();
|
|
expect(res.headers["content-type"]).toContain("application/json");
|
|
expect(res.body).toEqual({ data: { ok: true } });
|
|
});
|
|
|
|
it("delivers a tool failure as an in-band terminal error line", async () => {
|
|
const registry = stubRegistry(async (options) => {
|
|
options?.emitProgress?.({ index: 0, status: "pass" });
|
|
throw new Error("device went away");
|
|
});
|
|
handle = createHttpApp(registry);
|
|
|
|
// Headers are already on the wire when the tool fails, so the status stays
|
|
// 200 and the error rides the stream's terminal line.
|
|
const res = await supertest(handle.app)
|
|
.post("/tools/test-tool")
|
|
.set("Accept", "application/x-ndjson")
|
|
.send({})
|
|
.buffer(true)
|
|
.parse(collectText)
|
|
.expect(200);
|
|
|
|
const lines = parseLines(res.body as string);
|
|
expect(lines).toHaveLength(2);
|
|
expect(lines[0]).toEqual({ event: "progress", data: { index: 0, status: "pass" } });
|
|
expect(lines[1].event).toBe("error");
|
|
expect(String(lines[1].error)).toContain("device went away");
|
|
});
|
|
|
|
it("keeps plain-JSON status codes for failures before the invoke (unknown tool)", async () => {
|
|
handle = createHttpApp(stubRegistry(async () => ({ ok: true })));
|
|
|
|
// The 404 gate fires before the response commits to streaming.
|
|
const res = await supertest(handle.app)
|
|
.post("/tools/nope")
|
|
.set("Accept", "application/x-ndjson")
|
|
.send({})
|
|
.expect(404);
|
|
|
|
expect(res.body.error).toContain("nope");
|
|
});
|
|
});
|