Files
software-mansion__argent/packages/tool-server/test/flows/flow-chromium-tree.test.ts
Jakub Piasecki 551c781fb0 feat(flow): support running end-to-end flows without agent (#429)
# Flow E2E

This PR turns flows from replayable tool-call recordings into a
declarative, portable e2e test system that runs headlessly via `argent
flow run <name>` — no LLM in the loop, non-zero exit on failure,
suitable for CI.

Agents can still replay flows via `flow-execute` and get the full output
from each step after it finishes.

## Flow format

Flows are YAML files in `.argent/flows/`, now built from **directives**
interpreted by the runner rather than raw tool calls:

- `launch` — terminate + relaunch the app under test (per-platform app
id map supported); its presence marks a flow as **e2e** (controls its
own start state) vs. a **fragment** (runs against current state,
declares an `executionPrerequisite`, composable via `run:` with
cycle/depth guards).
- `tap`, `type` (focus handshake + optional submit), `scroll-to`
(momentum-free, increments by half the container, `within:` for nested
scrollers), `await`/`assert` (`visible`/`exists`/`hidden`/`text`
conditions), `wait`, `snapshot`, `echo`, and raw `tool:` as escape hatch
— still fully supported, so previously recorded flows keep working
unchanged.
- **Selectors** are `{ text, identifier, role }`; a bare string is a
loose selector resolving identifier-first, then visible text. Any
directive failure hard-stops the flow; later steps report `skip`.

## Execution engine

- Flows store no device id — the runner binds a device at replay
(explicit `--device`/`--platform` or the single booted one).
- Selector resolution runs against **platform-unified UI trees**: iOS
native view hierarchy (full testID coverage, gated on native-devtools
connecting), Android full accessibility hierarchy (full resource-id
coverage), Chromium CDP DOM, and the Vega automation toolkit. A shared
flatten/text-hoisting pass lets `text` assertions read what an
identified container visibly shows. Vega is remote-driven, so touch
directives fail with guidance to use `tv-remote`/`keyboard`.
- Steps wait for the tree to **settle** (two consecutive identical
reads) before acting; `launch` additionally gates on platform
tree-source readiness instead of silently degrading.

## Visual snapshots

`snapshot` diffs a full-resolution screenshot against baselines in
`.argent/flows/__baselines__/<flow>/`, keyed by platform + resolution,
with configurable `maxMismatch` and `--update-baselines`. The status bar
is pinned for the whole run (iOS `simctl status_bar`, Android demo mode)
so clocks/battery don't drive diffs. Baseline/current/diff images are
returned as artifacts on failure (and the written baseline on first
capture/update); a clean pass carries none, keeping reports and agent
context lean.

## Recording

The record-live workflow (`flow-start-recording` → `flow-add-step` →
`flow-finish-recording`) now emits directives automatically: coordinate
taps are rewritten to portable selector `tap:` steps (selector captured
from the pre-tap tree), a leading `restart-app` becomes `launch:`,
`flow-execute` of a sibling fragment becomes `run:`, and device ids are
stripped. The skill instructs the agent to polish the saved YAML
afterwards (fold tap+keyboard into `type:`, swipes-to-reach into
`scroll-to:`, awaits into sugar form).

## Also included

- `argent flow run` / `argent flow list` CLI with per-step report
rendering, artifact materialization, and `--json` output.
- New structured failure codes for flow validation, device resolution,
composition, and devtools readiness.
- Extensive unit test coverage (tree adapters, selector matching,
scrolling, composition, type focus, snapshots, status bar).
2026-07-13 17:15:01 +02:00

148 lines
5.7 KiB
TypeScript

import { describe, it, expect } from "vitest";
import { adaptChromiumTreeForFlows } from "../../src/tools/flows/flow-chromium-tree";
import { assertText, evaluateCondition, findAll } from "../../src/utils/ui-tree-match";
import type { DescribeNode } from "../../src/tools/describe/contract";
const FULL = { x: 0, y: 0, width: 1, height: 1 };
function el(partial: Partial<DescribeNode>): DescribeNode {
return { role: "div", frame: FULL, children: [], ...partial } as DescribeNode;
}
// A screen shaped like the CDP DOM walker's output for an RN-web app: a
// testID'd log container whose lines are child text nodes, a labelled button,
// a password input, and a row scrolled off-viewport (frame clamped to zero
// area by the walker).
function screen(): DescribeNode {
return el({
role: "html",
children: [
el({
identifier: "log-box",
frame: { x: 0.02, y: 0.68, width: 0.96, height: 0.2 },
children: [
el({ value: "Event log", frame: { x: 0.03, y: 0.7, width: 0.94, height: 0.02 } }),
el({
value: "[15:06:21] outer tap gesture",
frame: { x: 0.03, y: 0.73, width: 0.94, height: 0.02 },
}),
],
}),
el({
role: "button",
clickable: true,
frame: { x: 0.45, y: 0.3, width: 0.09, height: 0.04 },
children: [
el({ value: "Clear logs", frame: { x: 0.45, y: 0.3, width: 0.09, height: 0.04 } }),
],
}),
el({
role: "input",
identifier: "pw",
password: true,
label: "Enter password",
frame: { x: 0.1, y: 0.1, width: 0.5, height: 0.04 },
}),
el({ value: "Below the fold", frame: { x: 0.03, y: 1, width: 0.94, height: 0 } }),
],
});
}
describe("adaptChromiumTreeForFlows", () => {
it("hoists descendant text onto an identified container for text asserts", () => {
const tree = adaptChromiumTreeForFlows(screen());
const matches = findAll(tree, { identifier: "log-box" });
expect(matches).toHaveLength(1);
expect(assertText(matches[0]!)).toContain("outer tap gesture");
expect(evaluateCondition("text", "outer tap gesture", matches, "contains")).toBe(true);
});
it("scopes hoisted text to the nearest identified ancestor", () => {
const tree = adaptChromiumTreeForFlows(screen());
// log-box shields its lines: the root/screen must not swallow them, and
// sibling containers must not see them.
const button = findAll(tree, { role: "button" }).find((n) => n.clickable);
expect(button).toBeDefined();
expect(assertText(button!)).not.toContain("outer tap gesture");
});
it("keeps selector targeting on child text (tap by text still resolves)", () => {
const tree = adaptChromiumTreeForFlows(screen());
const matches = findAll(tree, { text: "Clear logs" });
expect(matches.length).toBeGreaterThan(0);
expect(matches[0]!.frame.height).toBeGreaterThan(0);
});
it("never hoists text from zero-area (off-viewport) nodes", () => {
const tree = adaptChromiumTreeForFlows(screen());
const walk = (n: DescribeNode): string[] => [n.subtreeText ?? "", ...n.children.flatMap(walk)];
expect(walk(tree).join(" ")).not.toContain("Below the fold");
});
it("shields a password field's text from bubbling upward", () => {
const inner = el({
identifier: "form",
frame: { x: 0, y: 0, width: 1, height: 0.5 },
children: [
el({
role: "input",
password: true,
label: "hunter2",
frame: { x: 0, y: 0.1, width: 1, height: 0.05 },
}),
],
});
const tree = adaptChromiumTreeForFlows(el({ role: "html", children: [inner] }));
const form = findAll(tree, { identifier: "form" });
expect(form).toHaveLength(1);
expect(assertText(form[0]!)).not.toContain("hunter2");
});
it("redacts a password leaf's label so a failing assert can't echo the secret", () => {
// The walker never reads a password's value into the label, but the leaf
// redacts too (defense in depth, mirroring the Android adapter's
// `[password]` placeholder) — assertReason echoes a leaf's text verbatim.
const tree = adaptChromiumTreeForFlows(
el({
role: "html",
children: [
el({
role: "input",
identifier: "pw",
password: true,
label: "hunter2",
frame: { x: 0.1, y: 0.1, width: 0.5, height: 0.04 },
}),
],
})
);
const pw = findAll(tree, { identifier: "pw" });
expect(pw).toHaveLength(1);
expect(pw[0]!.label).toBe("[password]");
expect(assertText(pw[0]!)).not.toContain("hunter2");
});
it("never emits the root as a leaf, even when <html> carries an identifier", () => {
// The walker reads id/data-testid off every element including <html>; the
// iOS/Android adapters iterate children only and never project their root.
// A projected root would be a full-screen leaf shielding the whole page's
// text, so a broad assert could pass spuriously.
const root = screen();
root.identifier = "app-root";
const tree = adaptChromiumTreeForFlows(root);
expect(findAll(tree, { identifier: "app-root" })).toHaveLength(0);
// The root's descendants still flatten normally.
expect(findAll(tree, { identifier: "log-box" })).toHaveLength(1);
});
it("drops pure scaffolding but keeps addressable descendants", () => {
const tree = adaptChromiumTreeForFlows(screen());
// Root wrapper: flat leaves under one synthetic Screen node.
expect(tree.role).toBe("Screen");
for (const child of tree.children) {
expect(child.children).toHaveLength(0);
expect(Boolean(child.identifier || child.label || child.value || child.clickable)).toBe(true);
}
});
});