Files
software-mansion__argent/packages/tool-server/test/flows/flow-tree-no-fallback.test.ts
Jakub Piasecki 551c781fb0 feat(flow): support running end-to-end flows without agent (#429)
# Flow E2E

This PR turns flows from replayable tool-call recordings into a
declarative, portable e2e test system that runs headlessly via `argent
flow run <name>` — no LLM in the loop, non-zero exit on failure,
suitable for CI.

Agents can still replay flows via `flow-execute` and get the full output
from each step after it finishes.

## Flow format

Flows are YAML files in `.argent/flows/`, now built from **directives**
interpreted by the runner rather than raw tool calls:

- `launch` — terminate + relaunch the app under test (per-platform app
id map supported); its presence marks a flow as **e2e** (controls its
own start state) vs. a **fragment** (runs against current state,
declares an `executionPrerequisite`, composable via `run:` with
cycle/depth guards).
- `tap`, `type` (focus handshake + optional submit), `scroll-to`
(momentum-free, increments by half the container, `within:` for nested
scrollers), `await`/`assert` (`visible`/`exists`/`hidden`/`text`
conditions), `wait`, `snapshot`, `echo`, and raw `tool:` as escape hatch
— still fully supported, so previously recorded flows keep working
unchanged.
- **Selectors** are `{ text, identifier, role }`; a bare string is a
loose selector resolving identifier-first, then visible text. Any
directive failure hard-stops the flow; later steps report `skip`.

## Execution engine

- Flows store no device id — the runner binds a device at replay
(explicit `--device`/`--platform` or the single booted one).
- Selector resolution runs against **platform-unified UI trees**: iOS
native view hierarchy (full testID coverage, gated on native-devtools
connecting), Android full accessibility hierarchy (full resource-id
coverage), Chromium CDP DOM, and the Vega automation toolkit. A shared
flatten/text-hoisting pass lets `text` assertions read what an
identified container visibly shows. Vega is remote-driven, so touch
directives fail with guidance to use `tv-remote`/`keyboard`.
- Steps wait for the tree to **settle** (two consecutive identical
reads) before acting; `launch` additionally gates on platform
tree-source readiness instead of silently degrading.

## Visual snapshots

`snapshot` diffs a full-resolution screenshot against baselines in
`.argent/flows/__baselines__/<flow>/`, keyed by platform + resolution,
with configurable `maxMismatch` and `--update-baselines`. The status bar
is pinned for the whole run (iOS `simctl status_bar`, Android demo mode)
so clocks/battery don't drive diffs. Baseline/current/diff images are
returned as artifacts on failure (and the written baseline on first
capture/update); a clean pass carries none, keeping reports and agent
context lean.

## Recording

The record-live workflow (`flow-start-recording` → `flow-add-step` →
`flow-finish-recording`) now emits directives automatically: coordinate
taps are rewritten to portable selector `tap:` steps (selector captured
from the pre-tap tree), a leading `restart-app` becomes `launch:`,
`flow-execute` of a sibling fragment becomes `run:`, and device ids are
stripped. The skill instructs the agent to polish the saved YAML
afterwards (fold tap+keyboard into `type:`, swipes-to-reach into
`scroll-to:`, awaits into sugar form).

## Also included

- `argent flow run` / `argent flow list` CLI with per-step report
rendering, artifact materialization, and `--json` output.
- New structured failure codes for flow validation, device resolution,
composition, and devtools readiness.
- Extensive unit test coverage (tree adapters, selector matching,
scrolling, composition, type focus, snapshots, status bar).
2026-07-13 17:15:01 +02:00

99 lines
3.7 KiB
TypeScript

import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import type { DeviceInfo, Registry } from "@argent/registry";
import type { DescribeTreeData } from "../../src/tools/describe/contract";
// Spy on the trimmed AX describe path: if fetchFlowTree ever fell back to it,
// the same flow would pass or fail with devtools availability instead of with
// what's on screen (the trimmed tree lacks testID nodes and hoisted
// subtreeText). These tests pin the contract that it hard-fails instead.
const describeIos = vi.fn(async (): Promise<DescribeTreeData> => {
throw new Error("describeIos must not be reached by a flow tree fetch");
});
vi.mock("../../src/tools/describe/platforms/ios", () => ({
describeIos: (...args: unknown[]) => describeIos(...(args as [])),
}));
import { fetchFlowTree } from "../../src/tools/flows/flow-tree";
import { createRunFlowTool, type FlowRunResult } from "../../src/tools/flows/flow-run";
import { serializeFlow } from "../../src/tools/flows/flow-utils";
const IOS_DEVICE = "00000000-0000-0000-0000-0000000000ab";
let tmpDir: string;
function device(platform: string): DeviceInfo {
return { platform, id: IOS_DEVICE, udid: IOS_DEVICE } as unknown as DeviceInfo;
}
// Registry whose service layer is down — the shape a run sees when native
// devtools never connected or dropped mid-run.
function deadRegistry(): Registry {
return {
invokeTool: vi.fn(async (id: string) => {
if (id === "list-devices") return { devices: [] };
return { ok: true };
}),
getTool: vi.fn(() => undefined),
resolveService: vi.fn(async () => {
throw new Error("service unavailable");
}),
} as unknown as Registry;
}
function asRun(r: FlowRunResult | { notice: string }): FlowRunResult {
if (!("steps" in r)) throw new Error(`expected a run result, got notice: ${r.notice}`);
return r;
}
beforeEach(async () => {
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-nofallback-"));
describeIos.mockClear();
});
afterEach(async () => {
await fs.rm(tmpDir, { recursive: true, force: true });
});
describe("fetchFlowTree without a full-hierarchy source", () => {
it("throws on iOS instead of degrading to the AX tree", async () => {
await expect(fetchFlowTree(deadRegistry(), device("ios"))).rejects.toThrow(
/native devtools is unavailable/
);
expect(describeIos).not.toHaveBeenCalled();
});
it("throws on Android instead of degrading to the trimmed uiautomator tree", async () => {
await expect(fetchFlowTree(deadRegistry(), device("android"))).rejects.toThrow(
/android devtools helper is unavailable/
);
});
it("fails an assert step with the tree-source reason, not a selector miss", async () => {
const dir = path.join(tmpDir, ".argent", "flows");
await fs.mkdir(dir, { recursive: true });
await fs.writeFile(
path.join(dir, "check.yaml"),
serializeFlow({
executionPrerequisite: "",
steps: [{ kind: "assert", condition: "visible", selector: { text: "Continue" } }],
}),
"utf8"
);
const result = asRun(
await createRunFlowTool(deadRegistry()).execute(
{},
{ name: "check", project_root: tmpDir, device: IOS_DEVICE }
)
);
expect(result.ok).toBe(false);
expect(result.steps.map((s) => `${s.kind}:${s.status}`)).toEqual(["assert:fail"]);
// The report names the outage — not a misleading "no element matched".
expect(result.steps[0].reason).toMatch(/could not read the UI tree/);
expect(result.steps[0].reason).toMatch(/native devtools is unavailable/);
expect(describeIos).not.toHaveBeenCalled();
});
});