mirror of
https://github.com/software-mansion/argent.git
synced 2026-09-14 19:27:14 +08:00
551c781fb0
# Flow E2E
This PR turns flows from replayable tool-call recordings into a
declarative, portable e2e test system that runs headlessly via `argent
flow run <name>` — no LLM in the loop, non-zero exit on failure,
suitable for CI.
Agents can still replay flows via `flow-execute` and get the full output
from each step after it finishes.
## Flow format
Flows are YAML files in `.argent/flows/`, now built from **directives**
interpreted by the runner rather than raw tool calls:
- `launch` — terminate + relaunch the app under test (per-platform app
id map supported); its presence marks a flow as **e2e** (controls its
own start state) vs. a **fragment** (runs against current state,
declares an `executionPrerequisite`, composable via `run:` with
cycle/depth guards).
- `tap`, `type` (focus handshake + optional submit), `scroll-to`
(momentum-free, increments by half the container, `within:` for nested
scrollers), `await`/`assert` (`visible`/`exists`/`hidden`/`text`
conditions), `wait`, `snapshot`, `echo`, and raw `tool:` as escape hatch
— still fully supported, so previously recorded flows keep working
unchanged.
- **Selectors** are `{ text, identifier, role }`; a bare string is a
loose selector resolving identifier-first, then visible text. Any
directive failure hard-stops the flow; later steps report `skip`.
## Execution engine
- Flows store no device id — the runner binds a device at replay
(explicit `--device`/`--platform` or the single booted one).
- Selector resolution runs against **platform-unified UI trees**: iOS
native view hierarchy (full testID coverage, gated on native-devtools
connecting), Android full accessibility hierarchy (full resource-id
coverage), Chromium CDP DOM, and the Vega automation toolkit. A shared
flatten/text-hoisting pass lets `text` assertions read what an
identified container visibly shows. Vega is remote-driven, so touch
directives fail with guidance to use `tv-remote`/`keyboard`.
- Steps wait for the tree to **settle** (two consecutive identical
reads) before acting; `launch` additionally gates on platform
tree-source readiness instead of silently degrading.
## Visual snapshots
`snapshot` diffs a full-resolution screenshot against baselines in
`.argent/flows/__baselines__/<flow>/`, keyed by platform + resolution,
with configurable `maxMismatch` and `--update-baselines`. The status bar
is pinned for the whole run (iOS `simctl status_bar`, Android demo mode)
so clocks/battery don't drive diffs. Baseline/current/diff images are
returned as artifacts on failure (and the written baseline on first
capture/update); a clean pass carries none, keeping reports and agent
context lean.
## Recording
The record-live workflow (`flow-start-recording` → `flow-add-step` →
`flow-finish-recording`) now emits directives automatically: coordinate
taps are rewritten to portable selector `tap:` steps (selector captured
from the pre-tap tree), a leading `restart-app` becomes `launch:`,
`flow-execute` of a sibling fragment becomes `run:`, and device ids are
stripped. The skill instructs the agent to polish the saved YAML
afterwards (fold tap+keyboard into `type:`, swipes-to-reach into
`scroll-to:`, awaits into sugar form).
## Also included
- `argent flow run` / `argent flow list` CLI with per-step report
rendering, artifact materialization, and `--json` output.
- New structured failure codes for flow validation, device resolution,
composition, and devtools readiness.
- Extensive unit test coverage (tree adapters, selector matching,
scrolling, composition, type focus, snapshots, status bar).
202 lines
6.8 KiB
TypeScript
202 lines
6.8 KiB
TypeScript
/**
|
|
* Teardown-path tests for runner-booted Chromium: `killChromiumByPort` must
|
|
* prefer the ChildProcess handle retained at boot — whose exitCode/signalCode
|
|
* guard lets killChildEscalating skip the delayed SIGKILL once the child has
|
|
* exited, so it can never land on a recycled pid — and only fall back to
|
|
* raw-pid signalling (with a signal-0 liveness re-probe before the SIGKILL)
|
|
* once the child has left the registry.
|
|
*/
|
|
|
|
import { describe, it, expect, vi, beforeEach, afterEach, beforeAll, afterAll } from "vitest";
|
|
import { EventEmitter } from "node:events";
|
|
import * as fs from "node:fs";
|
|
import * as http from "node:http";
|
|
import * as os from "node:os";
|
|
import * as path from "node:path";
|
|
|
|
const spawnMock = vi.fn();
|
|
|
|
vi.mock("node:child_process", async () => {
|
|
const actual = await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
|
return {
|
|
...actual,
|
|
spawn: (cmd: string, args: string[], opts: unknown) => spawnMock(cmd, args, opts),
|
|
};
|
|
});
|
|
// Keep trackChromiumPort from persisting booted ports to on-disk state.
|
|
vi.mock("../src/utils/chromium-discovery", () => ({
|
|
trackChromiumPort: vi.fn(),
|
|
untrackChromiumPort: vi.fn(),
|
|
}));
|
|
|
|
import { bootElectronApp, killChromiumByPort } from "../src/tools/devices/boot-electron";
|
|
|
|
const FAKE_PID = 4242;
|
|
|
|
interface FakeChild extends EventEmitter {
|
|
pid: number | undefined;
|
|
stderr: EventEmitter;
|
|
unref: () => void;
|
|
kill: ReturnType<typeof vi.fn>;
|
|
exitCode: number | null;
|
|
signalCode: NodeJS.Signals | null;
|
|
}
|
|
|
|
function makeFakeChild(): FakeChild {
|
|
const ee = new EventEmitter() as FakeChild;
|
|
ee.pid = FAKE_PID;
|
|
ee.stderr = new EventEmitter();
|
|
ee.unref = () => {};
|
|
ee.kill = vi.fn(() => true);
|
|
ee.exitCode = null;
|
|
ee.signalCode = null;
|
|
return ee;
|
|
}
|
|
|
|
let appDir: string;
|
|
beforeAll(() => {
|
|
// resolveLauncher() fs-checks the app path before spawn, so the test needs
|
|
// a real directory on disk. The spawn itself is mocked, so the contents
|
|
// don't matter — only the path's existence.
|
|
appDir = fs.mkdtempSync(path.join(os.tmpdir(), "argent-boot-electron-kill-test-"));
|
|
fs.writeFileSync(
|
|
path.join(appDir, "package.json"),
|
|
JSON.stringify({ name: "fake-electron-app", main: "main.js" })
|
|
);
|
|
fs.writeFileSync(path.join(appDir, "main.js"), "// fake\n");
|
|
});
|
|
afterAll(() => {
|
|
if (appDir) fs.rmSync(appDir, { recursive: true, force: true });
|
|
});
|
|
|
|
beforeEach(() => {
|
|
spawnMock.mockReset();
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.useRealTimers();
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
/**
|
|
* Minimal CDP stub: only /json/version, which is all waitForCdpReady probes.
|
|
* Booting against it lands the fake child in the boot-electron handle
|
|
* registry, which is the state under test.
|
|
*/
|
|
async function bootFakeChild(): Promise<{ child: FakeChild; port: number; close: () => void }> {
|
|
const child = makeFakeChild();
|
|
spawnMock.mockReturnValue(child);
|
|
|
|
const srv = http.createServer((req, res) => {
|
|
res.setHeader("content-type", "application/json");
|
|
if (req.url === "/json/version") {
|
|
res.end(JSON.stringify({ "Browser": "Chrome/Test", "Protocol-Version": "1.3" }));
|
|
return;
|
|
}
|
|
res.statusCode = 404;
|
|
res.end();
|
|
});
|
|
await new Promise<void>((resolve) => srv.listen(0, "127.0.0.1", resolve));
|
|
const { port } = srv.address() as { port: number };
|
|
|
|
await bootElectronApp({ appPath: appDir, port, readyTimeoutMs: 5000 });
|
|
return { child, port, close: () => srv.close() };
|
|
}
|
|
|
|
describe("killChromiumByPort — handle path", () => {
|
|
it("kills through the retained handle: SIGTERM, then SIGKILL once the grace period elapses with the child still running", async () => {
|
|
const { child, port, close } = await bootFakeChild();
|
|
try {
|
|
vi.useFakeTimers();
|
|
killChromiumByPort(port, FAKE_PID);
|
|
|
|
// The handle path was taken (child.kill, not process.kill on a raw pid).
|
|
expect(child.kill).toHaveBeenCalledTimes(1);
|
|
expect(child.kill).toHaveBeenCalledWith("SIGTERM");
|
|
|
|
// Child ignores the SIGTERM (exitCode stays null) → escalate after 2s.
|
|
vi.advanceTimersByTime(2000);
|
|
expect(child.kill).toHaveBeenCalledTimes(2);
|
|
expect(child.kill).toHaveBeenLastCalledWith("SIGKILL");
|
|
} finally {
|
|
close();
|
|
}
|
|
});
|
|
|
|
it("skips the delayed SIGKILL when the child exits within the grace period (the recycled-pid guard)", async () => {
|
|
const { child, port, close } = await bootFakeChild();
|
|
try {
|
|
vi.useFakeTimers();
|
|
killChromiumByPort(port, FAKE_PID);
|
|
expect(child.kill).toHaveBeenCalledWith("SIGTERM");
|
|
|
|
// The SIGTERM lands: the child exits before the escalation timer fires.
|
|
child.exitCode = 0;
|
|
vi.advanceTimersByTime(2000);
|
|
|
|
expect(child.kill).toHaveBeenCalledTimes(1); // no SIGKILL
|
|
} finally {
|
|
close();
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("killChromiumByPort — raw-pid fallback", () => {
|
|
it("falls back to pid signalling after the child's exit evicted the handle, and the liveness probe suppresses the SIGKILL", async () => {
|
|
const { child, port, close } = await bootFakeChild();
|
|
try {
|
|
// Natural exit (e.g. the user closed the window): the cleanup listener
|
|
// must drop the handle from the registry.
|
|
child.emit("exit", 0, null);
|
|
|
|
const killSpy = vi
|
|
.spyOn(process, "kill")
|
|
.mockImplementation((_pid: number, signal?: string | number) => {
|
|
if (signal === 0) {
|
|
// By probe time the process is gone.
|
|
const err = new Error("kill ESRCH") as NodeJS.ErrnoException;
|
|
err.code = "ESRCH";
|
|
throw err;
|
|
}
|
|
return true;
|
|
});
|
|
|
|
vi.useFakeTimers();
|
|
killChromiumByPort(port, FAKE_PID);
|
|
|
|
// Handle gone → raw-pid fallback, never the stale ChildProcess.
|
|
expect(child.kill).not.toHaveBeenCalled();
|
|
expect(killSpy).toHaveBeenCalledWith(FAKE_PID, "SIGTERM");
|
|
|
|
vi.advanceTimersByTime(2000);
|
|
|
|
// The 0-probe reported ESRCH → the SIGKILL must be suppressed so it
|
|
// cannot land on a recycled pid.
|
|
expect(killSpy).toHaveBeenCalledWith(FAKE_PID, 0);
|
|
const signals = killSpy.mock.calls.map((c) => c[1]);
|
|
expect(signals).not.toContain("SIGKILL");
|
|
} finally {
|
|
close();
|
|
}
|
|
});
|
|
|
|
it("escalates the fallback to SIGKILL when the pid is still alive at probe time", () => {
|
|
const killSpy = vi.spyOn(process, "kill").mockImplementation(() => true);
|
|
vi.useFakeTimers();
|
|
|
|
// Port never registered → straight to the pid fallback.
|
|
killChromiumByPort(59999, FAKE_PID);
|
|
expect(killSpy).toHaveBeenCalledWith(FAKE_PID, "SIGTERM");
|
|
|
|
vi.advanceTimersByTime(2000);
|
|
expect(killSpy).toHaveBeenCalledWith(FAKE_PID, 0);
|
|
expect(killSpy).toHaveBeenCalledWith(FAKE_PID, "SIGKILL");
|
|
});
|
|
|
|
it("is a no-op when no handle is registered and no pid is provided", () => {
|
|
const killSpy = vi.spyOn(process, "kill").mockImplementation(() => true);
|
|
killChromiumByPort(58888);
|
|
expect(killSpy).not.toHaveBeenCalled();
|
|
});
|
|
});
|