Files
software-mansion__argent/packages/tool-server/test/flows/flow-remote-recording.test.ts
Jakub Piasecki 551c781fb0 feat(flow): support running end-to-end flows without agent (#429)
# Flow E2E

This PR turns flows from replayable tool-call recordings into a
declarative, portable e2e test system that runs headlessly via `argent
flow run <name>` — no LLM in the loop, non-zero exit on failure,
suitable for CI.

Agents can still replay flows via `flow-execute` and get the full output
from each step after it finishes.

## Flow format

Flows are YAML files in `.argent/flows/`, now built from **directives**
interpreted by the runner rather than raw tool calls:

- `launch` — terminate + relaunch the app under test (per-platform app
id map supported); its presence marks a flow as **e2e** (controls its
own start state) vs. a **fragment** (runs against current state,
declares an `executionPrerequisite`, composable via `run:` with
cycle/depth guards).
- `tap`, `type` (focus handshake + optional submit), `scroll-to`
(momentum-free, increments by half the container, `within:` for nested
scrollers), `await`/`assert` (`visible`/`exists`/`hidden`/`text`
conditions), `wait`, `snapshot`, `echo`, and raw `tool:` as escape hatch
— still fully supported, so previously recorded flows keep working
unchanged.
- **Selectors** are `{ text, identifier, role }`; a bare string is a
loose selector resolving identifier-first, then visible text. Any
directive failure hard-stops the flow; later steps report `skip`.

## Execution engine

- Flows store no device id — the runner binds a device at replay
(explicit `--device`/`--platform` or the single booted one).
- Selector resolution runs against **platform-unified UI trees**: iOS
native view hierarchy (full testID coverage, gated on native-devtools
connecting), Android full accessibility hierarchy (full resource-id
coverage), Chromium CDP DOM, and the Vega automation toolkit. A shared
flatten/text-hoisting pass lets `text` assertions read what an
identified container visibly shows. Vega is remote-driven, so touch
directives fail with guidance to use `tv-remote`/`keyboard`.
- Steps wait for the tree to **settle** (two consecutive identical
reads) before acting; `launch` additionally gates on platform
tree-source readiness instead of silently degrading.

## Visual snapshots

`snapshot` diffs a full-resolution screenshot against baselines in
`.argent/flows/__baselines__/<flow>/`, keyed by platform + resolution,
with configurable `maxMismatch` and `--update-baselines`. The status bar
is pinned for the whole run (iOS `simctl status_bar`, Android demo mode)
so clocks/battery don't drive diffs. Baseline/current/diff images are
returned as artifacts on failure (and the written baseline on first
capture/update); a clean pass carries none, keeping reports and agent
context lean.

## Recording

The record-live workflow (`flow-start-recording` → `flow-add-step` →
`flow-finish-recording`) now emits directives automatically: coordinate
taps are rewritten to portable selector `tap:` steps (selector captured
from the pre-tap tree), a leading `restart-app` becomes `launch:`,
`flow-execute` of a sibling fragment becomes `run:`, and device ids are
stripped. The skill instructs the agent to polish the saved YAML
afterwards (fold tap+keyboard into `type:`, swipes-to-reach into
`scroll-to:`, awaits into sugar form).

## Also included

- `argent flow run` / `argent flow list` CLI with per-step report
rendering, artifact materialization, and `--json` output.
- New structured failure codes for flow validation, device resolution,
composition, and devtools readiness.
- Extensive unit test coverage (tree adapters, selector matching,
scrolling, composition, type focus, snapshots, status bar).
2026-07-13 17:15:01 +02:00

236 lines
8.4 KiB
TypeScript

import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import type { Registry, ToolContext } from "@argent/registry";
import { ArtifactStore, CLIENT_FILE_MARKER } from "@argent/registry";
import { flowStartRecordingTool } from "../../src/tools/flows/flow-start-recording";
import { flowInsertEchoTool } from "../../src/tools/flows/flow-insert-echo";
import { flowFinishRecordingTool } from "../../src/tools/flows/flow-finish-recording";
import { createFlowAddStepTool } from "../../src/tools/flows/flow-add-step";
import { createRunFlowTool, resolveFlowFilePath } from "../../src/tools/flows/flow-run";
import { flowReadPrerequisiteTool } from "../../src/tools/flows/flow-read-prerequisite";
import {
clearActiveFlow,
clearActiveProjectRoot,
parseFlow,
} from "../../src/tools/flows/flow-utils";
/**
* Remote-mode flow behavior: the agent's project_root does NOT exist on this
* host (the boundary probe says presentOnHost: false), so recording stays in
* memory and every mutating tool returns a client-write directive instead of
* touching this host's disk.
*/
// A path that exists on the (simulated) client but not on this "server".
const CLIENT_ROOT = path.join(os.tmpdir(), "definitely-not-on-this-host", "agent-project");
const CLIENT_FLOW_PATH = path.join(CLIENT_ROOT, ".argent", "flows", "remote-flow.yaml");
function remoteCtx(): ToolContext {
return {
artifacts: new ArtifactStore(),
fileInputs: {
project_root: { clientPath: CLIENT_ROOT, presentOnHost: false, viaUpload: false },
},
};
}
/** The ctx the boundary produces after materializing the client's uploaded flow YAML. */
function uploadCtx(): ToolContext {
return {
artifacts: new ArtifactStore(),
fileInputs: {
flow_file: { clientPath: CLIENT_FLOW_PATH, presentOnHost: false, viaUpload: true },
},
};
}
function createMockRegistry(tools: Record<string, { result: unknown }> = {}) {
return {
invokeTool: vi.fn(async (id: string) => {
const entry = tools[id];
if (!entry) throw new Error(`Tool "${id}" not found`);
return entry.result;
}),
getTool: vi.fn(() => undefined),
} as unknown as Registry;
}
beforeEach(() => {
clearActiveFlow();
});
afterEach(async () => {
clearActiveFlow();
clearActiveProjectRoot();
await fs.rm(CLIENT_ROOT, { recursive: true, force: true });
});
describe("flow recording with a remote client (probe miss)", () => {
it("start-recording returns a directive and writes nothing on this host", async () => {
const result = await flowStartRecordingTool.execute(
{},
{ name: "remote-flow", project_root: CLIENT_ROOT, executionPrerequisite: "Home" },
remoteCtx()
);
expect(result.savedTo).toMatchObject({
[CLIENT_FILE_MARKER]: true,
path: CLIENT_FLOW_PATH,
});
const directive = result.savedTo as { content: string };
expect(parseFlow(directive.content).executionPrerequisite).toBe("Home");
// The agent's directory layout must not be recreated on the server host.
await expect(fs.stat(CLIENT_ROOT)).rejects.toThrow();
});
it("add-step / add-echo accumulate in memory and return updated directives", async () => {
const registry = createMockRegistry({ tap: { result: { tapped: true } } });
const addStep = createFlowAddStepTool(registry);
await flowStartRecordingTool.execute(
{},
{ name: "remote-flow", project_root: CLIENT_ROOT, executionPrerequisite: "Home" },
remoteCtx()
);
await flowInsertEchoTool.execute({}, { message: "label" });
const stepResult = await addStep.execute({}, { command: "tap", args: '{"x":0.5}' });
const directive = stepResult.savedTo as { path: string; content: string };
expect(directive.path).toBe(CLIENT_FLOW_PATH);
expect(parseFlow(directive.content).steps).toEqual([
{ kind: "echo", message: "label" },
{ kind: "tool", name: "tap", args: { x: 0.5 } },
]);
await expect(fs.stat(CLIENT_ROOT)).rejects.toThrow();
});
it("finish-recording summarizes the in-memory flow and clears the session", async () => {
await flowStartRecordingTool.execute(
{},
{ name: "remote-flow", project_root: CLIENT_ROOT, executionPrerequisite: "Home" },
remoteCtx()
);
await flowInsertEchoTool.execute({}, { message: "only step" });
const result = await flowFinishRecordingTool.execute({}, {});
expect(result.steps).toBe(1);
expect(result.summary).toEqual(["1. echo: only step"]);
expect(result.path).toBe(CLIENT_FLOW_PATH);
expect(result.savedTo).toMatchObject({ [CLIENT_FILE_MARKER]: true });
await expect(flowFinishRecordingTool.execute({}, {})).rejects.toThrow("No active flow");
});
});
describe("flow replay with a boundary-resolved flow_file", () => {
it("flow-execute reads the resolved path instead of deriving from project_root", async () => {
// Simulates the server-side temp file the boundary materialized from the
// client's upload.
const uploaded = path.join(os.tmpdir(), `uploaded-flow-${Date.now()}.yaml`);
await fs.writeFile(
uploaded,
["executionPrerequisite: ''", "steps:", " - echo: from upload", ""].join("\n")
);
try {
const runFlow = createRunFlowTool(createMockRegistry());
const result = await runFlow.execute(
{},
{
name: "remote-flow",
project_root: CLIENT_ROOT,
flow_file: uploaded,
device: "00000000-0000-0000-0000-0000000000ab",
},
uploadCtx()
);
expect(result).toMatchObject({
flow: "remote-flow",
steps: [{ kind: "echo", status: "pass", message: "from upload" }],
});
} finally {
await fs.rm(uploaded, { force: true });
}
});
it("flow-read-prerequisite reads the resolved path", async () => {
const uploaded = path.join(os.tmpdir(), `uploaded-prereq-${Date.now()}.yaml`);
await fs.writeFile(
uploaded,
["executionPrerequisite: 'Device unlocked'", "steps: []", ""].join("\n")
);
try {
const result = await flowReadPrerequisiteTool.execute(
{},
{ name: "remote-flow", project_root: CLIENT_ROOT, flow_file: uploaded },
uploadCtx()
);
expect(result.executionPrerequisite).toBe("Device unlocked");
} finally {
await fs.rm(uploaded, { force: true });
}
});
});
describe("flow_file containment", () => {
const params = (flow_file: string) => ({
name: "remote-flow",
project_root: CLIENT_ROOT,
flow_file,
});
it("accepts the exact ${project_root}/.argent/flows/${name}.yaml path", () => {
expect(resolveFlowFilePath(params(CLIENT_FLOW_PATH))).toBe(CLIENT_FLOW_PATH);
});
it("accepts a boundary-materialized upload wherever the server put it", () => {
const uploaded = path.join(os.tmpdir(), "argent-file-input-abc", "remote-flow.yaml");
expect(
resolveFlowFilePath(params(uploaded), {
clientPath: CLIENT_FLOW_PATH,
presentOnHost: false,
viaUpload: true,
})
).toBe(uploaded);
});
it("rejects a relative flow_file", () => {
expect(() => resolveFlowFilePath(params(".argent/flows/remote-flow.yaml"))).toThrow(
"Invalid flow_file"
);
});
it('rejects ".." traversal even when it resolves back to the flows dir', () => {
// Raw concatenation — path.join would collapse the ".." before the check.
const sneaky = `${CLIENT_ROOT}/.argent/flows/../flows/remote-flow.yaml`;
expect(() => resolveFlowFilePath(params(sneaky))).toThrow("Invalid flow_file");
});
it("rejects an absolute path outside the project's flows dir", () => {
expect(() => resolveFlowFilePath(params("/etc/anything.yaml"))).toThrow("Invalid flow_file");
// A different flow's file under the right dir is not this flow's path either.
expect(() =>
resolveFlowFilePath(params(path.join(CLIENT_ROOT, ".argent", "flows", "other.yaml")))
).toThrow("Invalid flow_file");
});
it("flow-execute refuses an out-of-project flow_file without reading it", async () => {
const runFlow = createRunFlowTool(createMockRegistry());
await expect(
runFlow.execute(
{},
{
name: "remote-flow",
project_root: CLIENT_ROOT,
flow_file: "/etc/anything.yaml",
device: "00000000-0000-0000-0000-0000000000ab",
}
)
).rejects.toThrow("Invalid flow_file");
});
});