Files
software-mansion__argent/packages/tool-server/test/await-screen-idle.test.ts
Jakub Piasecki 551c781fb0 feat(flow): support running end-to-end flows without agent (#429)
# Flow E2E

This PR turns flows from replayable tool-call recordings into a
declarative, portable e2e test system that runs headlessly via `argent
flow run <name>` — no LLM in the loop, non-zero exit on failure,
suitable for CI.

Agents can still replay flows via `flow-execute` and get the full output
from each step after it finishes.

## Flow format

Flows are YAML files in `.argent/flows/`, now built from **directives**
interpreted by the runner rather than raw tool calls:

- `launch` — terminate + relaunch the app under test (per-platform app
id map supported); its presence marks a flow as **e2e** (controls its
own start state) vs. a **fragment** (runs against current state,
declares an `executionPrerequisite`, composable via `run:` with
cycle/depth guards).
- `tap`, `type` (focus handshake + optional submit), `scroll-to`
(momentum-free, increments by half the container, `within:` for nested
scrollers), `await`/`assert` (`visible`/`exists`/`hidden`/`text`
conditions), `wait`, `snapshot`, `echo`, and raw `tool:` as escape hatch
— still fully supported, so previously recorded flows keep working
unchanged.
- **Selectors** are `{ text, identifier, role }`; a bare string is a
loose selector resolving identifier-first, then visible text. Any
directive failure hard-stops the flow; later steps report `skip`.

## Execution engine

- Flows store no device id — the runner binds a device at replay
(explicit `--device`/`--platform` or the single booted one).
- Selector resolution runs against **platform-unified UI trees**: iOS
native view hierarchy (full testID coverage, gated on native-devtools
connecting), Android full accessibility hierarchy (full resource-id
coverage), Chromium CDP DOM, and the Vega automation toolkit. A shared
flatten/text-hoisting pass lets `text` assertions read what an
identified container visibly shows. Vega is remote-driven, so touch
directives fail with guidance to use `tv-remote`/`keyboard`.
- Steps wait for the tree to **settle** (two consecutive identical
reads) before acting; `launch` additionally gates on platform
tree-source readiness instead of silently degrading.

## Visual snapshots

`snapshot` diffs a full-resolution screenshot against baselines in
`.argent/flows/__baselines__/<flow>/`, keyed by platform + resolution,
with configurable `maxMismatch` and `--update-baselines`. The status bar
is pinned for the whole run (iOS `simctl status_bar`, Android demo mode)
so clocks/battery don't drive diffs. Baseline/current/diff images are
returned as artifacts on failure (and the written baseline on first
capture/update); a clean pass carries none, keeping reports and agent
context lean.

## Recording

The record-live workflow (`flow-start-recording` → `flow-add-step` →
`flow-finish-recording`) now emits directives automatically: coordinate
taps are rewritten to portable selector `tap:` steps (selector captured
from the pre-tap tree), a leading `restart-app` becomes `launch:`,
`flow-execute` of a sibling fragment becomes `run:`, and device ids are
stripped. The skill instructs the agent to polish the saved YAML
afterwards (fold tap+keyboard into `type:`, swipes-to-reach into
`scroll-to:`, awaits into sugar form).

## Also included

- `argent flow run` / `argent flow list` CLI with per-step report
rendering, artifact materialization, and `--json` output.
- New structured failure codes for flow validation, device resolution,
composition, and devtools readiness.
- Extensive unit test coverage (tree adapters, selector matching,
scrolling, composition, type focus, snapshots, status bar).
2026-07-13 17:15:01 +02:00

114 lines
4.1 KiB
TypeScript

import { beforeEach, describe, expect, it, vi } from "vitest";
import type { AXServiceApi, AXDescribeResponse } from "../src/blueprints/ax-service";
import { createAwaitScreenIdleTool } from "../src/tools/await-screen-idle";
import { __primeDepCacheForTests, __resetDepCacheForTests } from "../src/utils/check-deps";
// execute() probes the target's form factor (isTvOsSimulator) before polling —
// a real `xcrun simctl list` that never caches for this fake UDID, so it re-runs
// on every test and takes seconds under the parallel suite load. The device here
// is a plain phone shape, so pin the probe to false and keep the rest real.
vi.mock("../src/utils/ios-devices", async () => {
const actual = await vi.importActual<typeof import("../src/utils/ios-devices")>(
"../src/utils/ios-devices"
);
return { ...actual, isTvOsSimulator: async () => false };
});
const IOS_UDID = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA";
const FRAME = { x: 0.1, y: 0.4, width: 0.8, height: 0.05 };
// AX service that walks `responses` one per call, repeating the last — lets a
// test simulate a screen that is blank, then renders, then holds still.
function makeSequencedAXService(responses: AXDescribeResponse[]): AXServiceApi {
let i = 0;
return {
degraded: false,
describe: async () => responses[Math.min(i++, responses.length - 1)],
alertCheck: async () => false,
ping: async () => true,
};
}
function axResponse(elements: AXDescribeResponse["elements"]): AXDescribeResponse {
return { alertVisible: false, screenFrame: { width: 440, height: 956 }, elements };
}
function iosRegistry(ax: AXServiceApi) {
return {
resolveService: vi.fn(async (urn: string) => {
if (urn.startsWith("AXService:")) return ax;
throw new Error(`unexpected service: ${urn}`);
}),
} as any;
}
const content = () => axResponse([{ label: "Settings", frame: FRAME, traits: ["button"] }]);
describe("await-screen-idle tool", () => {
beforeEach(() => {
__resetDepCacheForTests();
__primeDepCacheForTests(["xcrun", "adb"]);
});
it("exposes the await-screen-idle id", () => {
expect(createAwaitScreenIdleTool(iosRegistry({} as AXServiceApi)).id).toBe("await-screen-idle");
});
it("settles once content renders and holds still", async () => {
// blank, then the same content on every later poll → stable
const tool = createAwaitScreenIdleTool(
iosRegistry(makeSequencedAXService([axResponse([]), content()]))
);
const result = await tool.execute(
{},
{ udid: IOS_UDID, timeoutMs: 2000, pollIntervalMs: 10, minStableMs: 30 }
);
expect(result.settled).toBe(true);
expect(result.waitedMs).toBeGreaterThanOrEqual(30);
expect(result.waitedMs).toBeLessThan(2000);
expect(result.polls).toBeGreaterThan(1);
});
it("does not settle while the screen stays blank (times out)", async () => {
const tool = createAwaitScreenIdleTool(iosRegistry(makeSequencedAXService([axResponse([])])));
const result = await tool.execute(
{},
{ udid: IOS_UDID, timeoutMs: 60, pollIntervalMs: 10, minStableMs: 30 }
);
expect(result.settled).toBe(false);
expect(result.waitedMs).toBeGreaterThanOrEqual(60);
});
it("does not settle while content keeps changing (times out)", async () => {
// a different label every poll never holds for minStableMs
const changing = Array.from({ length: 30 }, (_, i) =>
axResponse([{ label: `item-${i}`, frame: FRAME, traits: ["button"] }])
);
const tool = createAwaitScreenIdleTool(iosRegistry(makeSequencedAXService(changing)));
const result = await tool.execute(
{},
{ udid: IOS_UDID, timeoutMs: 80, pollIntervalMs: 5, minStableMs: 40 }
);
expect(result.settled).toBe(false);
expect(result.waitedMs).toBeGreaterThanOrEqual(80);
});
it("settles on the first non-empty read when minStableMs is 0", async () => {
const tool = createAwaitScreenIdleTool(iosRegistry(makeSequencedAXService([content()])));
const result = await tool.execute(
{},
{ udid: IOS_UDID, timeoutMs: 2000, pollIntervalMs: 50, minStableMs: 0 }
);
expect(result.settled).toBe(true);
expect(result.polls).toBe(1);
});
});