mirror of
https://github.com/software-mansion/argent.git
synced 2026-09-14 19:27:14 +08:00
Hubert Gancarczyk
283bc76beb
`flow-visual.ts` ships five evidence payloads and only `snapshot-baseline-missing` was proven. Worse, the one relevant assertion was LOOSENED: `toHaveBeenCalledWith(env, cropOn)` became `(env, cropOn, expect.anything())`, which cannot distinguish the evidence sink from any object at all — and the `waitForFrame` mock ignored the third argument and never wrote to it, so the payload derived from that sink was never exercised. The mock now writes to the sink the way the real call does, the assertion pins the sink's shape and that it was written, and `snapshot-diff`, `snapshot-dimension-mismatch`, `snapshot-crop-empty` and both arms of the cropOn miss are asserted. The miss splits three ways, which the prose-only tests could not see: no readable tree is `tree-source-unavailable` (the selector was never evaluated), a readable tree without the element is `selector-not-found`, and a zero-area match is `selector-not-visible`. Two assertions in flow-failure-budget also allowed more than the budget they guard: `FLOW_FAILURE_BYTE_LIMIT * 2` accepted 47 KB from a 24 KB cap and was the only assertion on the trimmed size, and `FIELD_BYTE_LIMIT + 8` cited an `it.fails` that no longer exists. Both now assert the real limit, which the implementation already meets.
798 lines
32 KiB
TypeScript
798 lines
32 KiB
TypeScript
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||
import { createHash } from "node:crypto";
|
||
import * as fs from "node:fs/promises";
|
||
import * as os from "node:os";
|
||
import * as path from "node:path";
|
||
import { PNG } from "pngjs";
|
||
import { runSnapshot } from "../../src/tools/flows/flow-visual";
|
||
import { ArtifactStore } from "../../src/artifacts";
|
||
import {
|
||
diffPngFiles,
|
||
type DiffPngFilesOptions,
|
||
} from "../../src/tools/screenshot-diff/screenshot-diff";
|
||
import {
|
||
settleTree,
|
||
invokeOnDevice,
|
||
waitForFrame,
|
||
type ActionEnv,
|
||
} from "../../src/tools/flows/flow-actions";
|
||
import { redirectTmpdir } from "../helpers/tmpdir-env";
|
||
|
||
// Stub settle + capture so the tests exercise only the baseline write/diff decision.
|
||
const h = vi.hoisted(() => ({
|
||
shotPath: "",
|
||
mismatchPercentage: 0,
|
||
writeContextDiff: false,
|
||
/** Set by the differ mock: the context diff it wrote inside outputDir. */
|
||
contextDiffPath: "",
|
||
/** Set by the differ mock: the scratch outputDir runSnapshot handed it. */
|
||
outputDir: "",
|
||
/** Set by the differ mock: the currentPath it was asked to compare. */
|
||
diffCurrentPath: "",
|
||
/** Set by the differ mock: the top-mask policy it was passed. */
|
||
diffTopMask: "" as "" | NonNullable<DiffPngFilesOptions["topMask"]>,
|
||
/** Set by the differ mock: the normalizeSizes option it was passed. */
|
||
diffNormalizeSizes: undefined as boolean | undefined,
|
||
/** What the waitForFrame mock resolves a cropOn selector to. */
|
||
cropFrame: undefined as
|
||
| undefined
|
||
| "aborted"
|
||
| { x: number; y: number; width: number; height: number },
|
||
/** When set, the waitForFrame mock rejects with this (a tree-source outage). */
|
||
cropFrameError: null as null | Error,
|
||
/** The tree the waitForFrame mock publishes to its sink; null leaves it unread. */
|
||
cropSinkTree: null as null | Record<string, unknown>,
|
||
dimensionMismatch: null as null | {
|
||
expected: { width: number; height: number };
|
||
actual: { width: number; height: number };
|
||
},
|
||
}));
|
||
|
||
vi.mock("../../src/tools/flows/flow-actions", async (importOriginal) => ({
|
||
// The real offscreenHint, evidence builder, sink factory and timeout
|
||
// constant: cropOn failures must surface the directives' standard
|
||
// not-found reason AND their standard classified evidence, so the tests
|
||
// assert against the real implementations.
|
||
...(await importOriginal<typeof import("../../src/tools/flows/flow-actions")>()),
|
||
settleTree: vi.fn(async () => ({})),
|
||
invokeOnDevice: vi.fn(async () => ({ image: { hostPath: h.shotPath } })),
|
||
// Writes to the sink, as the real `waitForFrame` does: the cropOn-miss
|
||
// evidence (its code, its counts, its element list) is DERIVED from what
|
||
// lands there, so a mock that ignored the third argument proved nothing
|
||
// about the payload the report actually carries.
|
||
waitForFrame: vi.fn(async (_env: unknown, _sel: unknown, sink?: { attempts: number }) => {
|
||
if (sink) {
|
||
sink.attempts++;
|
||
if (h.cropSinkTree !== null) {
|
||
Object.assign(sink, { tree: h.cropSinkTree, source: "native-devtools", readAt: 1 });
|
||
}
|
||
}
|
||
if (h.cropFrameError) throw h.cropFrameError;
|
||
return h.cropFrame;
|
||
}),
|
||
}));
|
||
|
||
vi.mock("../../src/tools/screenshot-diff/screenshot-diff", () => ({
|
||
diffPngFiles: vi.fn(async (options: DiffPngFilesOptions) => {
|
||
h.outputDir = options.outputDir;
|
||
h.diffCurrentPath = options.currentPath;
|
||
h.diffTopMask = options.topMask ?? "status-bar";
|
||
h.diffNormalizeSizes = options.normalizeSizes;
|
||
// The real differ bails before writing anything on a dimension mismatch.
|
||
if (h.dimensionMismatch) {
|
||
return { mismatchPercentage: 0, dimensionMismatch: h.dimensionMismatch };
|
||
}
|
||
// Emulate the real differ: the full-res diff always lands in outputDir,
|
||
// the downscaled context diff only when a test asks for one.
|
||
const { writeFile } = await import("node:fs/promises");
|
||
const { join } = await import("node:path");
|
||
await writeFile(join(options.outputDir, "shot-diff.png"), Buffer.alloc(4));
|
||
let contextDiffPath: string | undefined;
|
||
if (h.writeContextDiff) {
|
||
contextDiffPath = join(options.outputDir, "shot-context-diff.png");
|
||
await writeFile(contextDiffPath, Buffer.alloc(4));
|
||
h.contextDiffPath = contextDiffPath;
|
||
}
|
||
return { mismatchPercentage: h.mismatchPercentage, contextDiffPath };
|
||
}),
|
||
}));
|
||
|
||
const env = {
|
||
device: { platform: "ios", id: "SIM" },
|
||
signal: undefined,
|
||
ctx: { artifacts: new ArtifactStore() },
|
||
} as unknown as ActionEnv;
|
||
|
||
let tmpDir: string;
|
||
|
||
/** Minimal PNG stand-in: runSnapshot reads only the IHDR width/height bytes. */
|
||
async function writeFakePng(file: string, w = 390, h_ = 844): Promise<void> {
|
||
const buf = Buffer.alloc(24);
|
||
buf.writeUInt32BE(w, 16);
|
||
buf.writeUInt32BE(h_, 20);
|
||
await fs.writeFile(file, buf);
|
||
}
|
||
|
||
/** Real PNG for the cropOn tests — the crop decodes actual pixel data. */
|
||
async function writeRealPng(file: string, w: number, h_: number): Promise<void> {
|
||
const png = new PNG({ width: w, height: h_ });
|
||
png.data.fill(128);
|
||
await fs.writeFile(file, PNG.sync.write(png));
|
||
}
|
||
|
||
async function pngSize(file: string): Promise<{ w: number; h: number }> {
|
||
const png = PNG.sync.read(await fs.readFile(file));
|
||
return { w: png.width, h: png.height };
|
||
}
|
||
|
||
/**
|
||
* Coordinate-encoded PNG: every pixel's channels name its own position
|
||
* (r = x, g = y, b = x + y), so a test can assert exactly WHICH region of the
|
||
* capture a crop contains — a transposed or shifted rect carries the wrong
|
||
* coordinates, where a uniform fill would make wrong pixels look right.
|
||
*/
|
||
async function writeCoordPng(file: string, w: number, h_: number): Promise<void> {
|
||
const png = new PNG({ width: w, height: h_ });
|
||
for (let y = 0; y < h_; y++) {
|
||
for (let x = 0; x < w; x++) {
|
||
const i = (y * w + x) * 4;
|
||
png.data[i] = x & 0xff;
|
||
png.data[i + 1] = y & 0xff;
|
||
png.data[i + 2] = (x + y) & 0xff;
|
||
png.data[i + 3] = 255;
|
||
}
|
||
}
|
||
await fs.writeFile(file, PNG.sync.write(png));
|
||
}
|
||
|
||
async function pngPixel(file: string, x: number, y: number): Promise<[number, number, number]> {
|
||
const png = PNG.sync.read(await fs.readFile(file));
|
||
const i = (y * png.width + x) * 4;
|
||
return [png.data[i], png.data[i + 1], png.data[i + 2]];
|
||
}
|
||
|
||
function opts(overrides: Partial<Parameters<typeof runSnapshot>[1]> = {}) {
|
||
return {
|
||
flowsDir: tmpDir,
|
||
flowName: "checkout",
|
||
name: "home",
|
||
maxMismatch: 0.5,
|
||
updateBaselines: false,
|
||
appIdentity: "/apps/app-a",
|
||
seenKeys: new Map<string, string>(),
|
||
...overrides,
|
||
};
|
||
}
|
||
|
||
const baselinePath = () => path.join(tmpDir, "__baselines__", "checkout", "home__ios-390x844.png");
|
||
|
||
beforeEach(async () => {
|
||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "flow-visual-"));
|
||
h.shotPath = path.join(tmpDir, "shot.png");
|
||
h.mismatchPercentage = 0;
|
||
h.writeContextDiff = false;
|
||
h.contextDiffPath = "";
|
||
h.outputDir = "";
|
||
h.diffCurrentPath = "";
|
||
h.diffTopMask = "";
|
||
h.diffNormalizeSizes = undefined;
|
||
h.cropFrame = undefined;
|
||
h.cropFrameError = null;
|
||
h.cropSinkTree = null;
|
||
h.dimensionMismatch = null;
|
||
await writeFakePng(h.shotPath);
|
||
});
|
||
afterEach(async () => {
|
||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||
});
|
||
|
||
describe("runSnapshot baselines", () => {
|
||
it("fails a missing baseline without seeding one", async () => {
|
||
const r = await runSnapshot(env, opts());
|
||
|
||
expect(r.status).toBe("fail");
|
||
expect(r.reason).toContain('no baseline for "home"');
|
||
expect(r.reason).toContain("--update-baselines");
|
||
// Nothing written: seeding on failure would make this unreviewed capture
|
||
// the truth a re-run silently passes against.
|
||
await expect(fs.access(baselinePath())).rejects.toThrow();
|
||
expect(r.artifacts?.current).toMatchObject({ hostPath: h.shotPath });
|
||
expect(r.artifacts?.baseline).toBeUndefined();
|
||
});
|
||
|
||
it("writes a missing baseline and passes under updateBaselines", async () => {
|
||
const r = await runSnapshot(env, opts({ updateBaselines: true }));
|
||
|
||
expect(r.status).toBe("pass");
|
||
expect(r.reason).toContain("baseline written");
|
||
await expect(fs.access(baselinePath())).resolves.toBeUndefined();
|
||
// The baseline travels as an artifact handle, not a raw host path.
|
||
expect(r.artifacts?.baseline).toMatchObject({
|
||
__argentArtifact: true,
|
||
hostPath: baselinePath(),
|
||
mimeType: "image/png",
|
||
});
|
||
// Full-screen keys carry no `-crop-` suffix — that is cropOn-only identity.
|
||
expect(r.snapshotKey).toBe("home__ios-390x844");
|
||
expect(r.snapshotKey).not.toContain("-crop-");
|
||
});
|
||
|
||
it("refreshes an existing baseline under updateBaselines", async () => {
|
||
await fs.mkdir(path.dirname(baselinePath()), { recursive: true });
|
||
await writeFakePng(baselinePath());
|
||
|
||
const r = await runSnapshot(env, opts({ updateBaselines: true }));
|
||
|
||
expect(r.status).toBe("pass");
|
||
expect(r.reason).toContain("baseline updated");
|
||
});
|
||
|
||
it("diffs against an existing baseline", async () => {
|
||
await fs.mkdir(path.dirname(baselinePath()), { recursive: true });
|
||
await writeFakePng(baselinePath());
|
||
|
||
const r = await runSnapshot(env, opts());
|
||
|
||
expect(r.status).toBe("pass");
|
||
expect(r.reason).toContain("diff 0.00%");
|
||
expect(h.diffTopMask).toBe("status-bar");
|
||
// Full-screen keeps the differ's default scale normalization (NOT false).
|
||
expect(h.diffNormalizeSizes).toBeUndefined();
|
||
// A clean pass carries no artifacts — there is nothing to look at, and
|
||
// handles would make renderers fetch two full-res PNGs just to print paths.
|
||
expect(r.artifacts).toBeUndefined();
|
||
expect(r.snapshotKey).toBeUndefined();
|
||
});
|
||
|
||
it("fails a dimension-mismatch bail instead of passing its 0% mismatch", async () => {
|
||
await fs.mkdir(path.dirname(baselinePath()), { recursive: true });
|
||
await writeFakePng(baselinePath());
|
||
// Full-screen keeps scale normalization, so the real differ only bails on a
|
||
// genuinely different aspect ratio — e.g. a rotated baseline vs the capture.
|
||
h.dimensionMismatch = {
|
||
expected: { width: 844, height: 390 },
|
||
actual: { width: 390, height: 844 },
|
||
};
|
||
|
||
const r = await runSnapshot(env, opts());
|
||
|
||
expect(r.status).toBe("fail");
|
||
expect(r.reason).toContain("844x390");
|
||
expect(r.reason).toContain("390x844");
|
||
expect(r.reason).toContain("nothing was compared");
|
||
expect(r.artifacts?.baseline).toMatchObject({ __argentArtifact: true });
|
||
expect(r.artifacts?.current).toMatchObject({ hostPath: h.shotPath });
|
||
// The classified payload, not just the prose: `dimensions` is what a
|
||
// renderer prints and what a repair loop branches on.
|
||
expect(r.evidence).toMatchObject({
|
||
code: "snapshot-dimension-mismatch",
|
||
expected: { kind: "snapshot", maxMismatch: 0.5 },
|
||
observation: {
|
||
dimensions: { expected: { width: 844, height: 390 }, actual: { width: 390, height: 844 } },
|
||
},
|
||
});
|
||
});
|
||
|
||
it("fails an over-threshold diff and exposes the context diff as an artifact", async () => {
|
||
await fs.mkdir(path.dirname(baselinePath()), { recursive: true });
|
||
await writeFakePng(baselinePath());
|
||
h.mismatchPercentage = 3.1;
|
||
h.writeContextDiff = true;
|
||
|
||
const r = await runSnapshot(env, opts());
|
||
|
||
expect(r.status).toBe("fail");
|
||
expect(r.reason).toContain("diff 3.10% > 0.5%");
|
||
// The key an exporter (CLI --output) names the three roles by.
|
||
expect(r.snapshotKey).toBe("home__ios-390x844");
|
||
expect(r.artifacts?.baseline).toMatchObject({ __argentArtifact: true });
|
||
expect(r.artifacts?.current).toMatchObject({ hostPath: h.shotPath });
|
||
expect(r.artifacts?.diff).toMatchObject({
|
||
__argentArtifact: true,
|
||
hostPath: h.contextDiffPath,
|
||
filename: "home__ios-390x844-diff.png",
|
||
});
|
||
// The measured mismatch reaches the payload, not only the prose — it is
|
||
// what `actual:` renders and what tells an author how far off they are.
|
||
expect(r.evidence).toMatchObject({
|
||
code: "snapshot-diff",
|
||
expected: { kind: "snapshot", snapshotKey: "home__ios-390x844", maxMismatch: 0.5 },
|
||
observation: { mismatchPercentage: 3.1 },
|
||
});
|
||
});
|
||
|
||
it("fails without a diff artifact when the differ produced no context image", async () => {
|
||
await fs.mkdir(path.dirname(baselinePath()), { recursive: true });
|
||
await writeFakePng(baselinePath());
|
||
h.mismatchPercentage = 100;
|
||
|
||
const r = await runSnapshot(env, opts());
|
||
|
||
expect(r.status).toBe("fail");
|
||
expect(r.artifacts?.baseline).toMatchObject({ __argentArtifact: true });
|
||
expect(r.artifacts?.current).toMatchObject({ hostPath: h.shotPath });
|
||
expect(r.artifacts?.diff).toBeUndefined();
|
||
});
|
||
});
|
||
|
||
describe("runSnapshot cross-app key collision", () => {
|
||
// The run already captured this key from another app — flow-run's seenKeys
|
||
// record, pre-seeded the way a prior snapshot step would leave it.
|
||
const seenFromAppB = () => new Map([["home__ios-390x844", "/apps/app-b"]]);
|
||
|
||
it("fails a key already captured from a different app instead of overwriting its baseline", async () => {
|
||
await fs.mkdir(path.dirname(baselinePath()), { recursive: true });
|
||
await fs.writeFile(baselinePath(), Buffer.from("app-b pixels"));
|
||
|
||
const r = await runSnapshot(env, opts({ updateBaselines: true, seenKeys: seenFromAppB() }));
|
||
|
||
expect(r.status).toBe("fail");
|
||
expect(r.reason).toContain(
|
||
'snapshot "home" was already captured in this run from a different app (/apps/app-b)'
|
||
);
|
||
expect(r.reason).toContain("home__ios-390x844.png");
|
||
expect(r.reason).toContain("distinct names");
|
||
expect(r.snapshotKey).toBe("home__ios-390x844");
|
||
// The other app's baseline survived — updateBaselines wrote nothing.
|
||
await expect(fs.readFile(baselinePath(), "utf8")).resolves.toBe("app-b pixels");
|
||
});
|
||
|
||
it("fails before comparing anything in plain compare mode", async () => {
|
||
await fs.mkdir(path.dirname(baselinePath()), { recursive: true });
|
||
await writeFakePng(baselinePath());
|
||
vi.mocked(diffPngFiles).mockClear();
|
||
|
||
const r = await runSnapshot(env, opts({ seenKeys: seenFromAppB() }));
|
||
|
||
expect(r.status).toBe("fail");
|
||
expect(r.reason).toContain("already captured in this run from a different app");
|
||
expect(vi.mocked(diffPngFiles)).not.toHaveBeenCalled();
|
||
});
|
||
|
||
it("allows recapturing a key from the same app", async () => {
|
||
await fs.mkdir(path.dirname(baselinePath()), { recursive: true });
|
||
await writeFakePng(baselinePath());
|
||
|
||
const r = await runSnapshot(
|
||
env,
|
||
opts({ seenKeys: new Map([["home__ios-390x844", "/apps/app-a"]]) })
|
||
);
|
||
|
||
expect(r.status).toBe("pass");
|
||
});
|
||
});
|
||
|
||
describe("runSnapshot settle", () => {
|
||
it("proceeds to capture when the tree source is down", async () => {
|
||
// settleTree throws when every read attempt failed (native devtools
|
||
// disconnected). The capture reads pixels, not the tree — the snapshot
|
||
// must still capture and compare instead of reporting an error.
|
||
vi.mocked(settleTree).mockRejectedValueOnce(new Error("native devtools is unavailable"));
|
||
vi.mocked(invokeOnDevice).mockClear();
|
||
await fs.mkdir(path.dirname(baselinePath()), { recursive: true });
|
||
await writeFakePng(baselinePath());
|
||
|
||
const r = await runSnapshot(env, opts());
|
||
|
||
expect(r.status).toBe("pass");
|
||
expect(vi.mocked(invokeOnDevice)).toHaveBeenCalledWith(env, "screenshot", expect.anything());
|
||
});
|
||
|
||
it("skips without capturing when the run was aborted during settle", async () => {
|
||
vi.mocked(settleTree).mockResolvedValueOnce(undefined);
|
||
vi.mocked(invokeOnDevice).mockClear();
|
||
const abortedEnv = { ...env, signal: { aborted: true } } as unknown as ActionEnv;
|
||
|
||
const r = await runSnapshot(abortedEnv, opts());
|
||
|
||
expect(r.status).toBe("skip");
|
||
expect(r.reason).toContain("aborted");
|
||
expect(vi.mocked(invokeOnDevice)).not.toHaveBeenCalled();
|
||
});
|
||
});
|
||
|
||
describe("runSnapshot diff-dir cleanup", () => {
|
||
const seedBaseline = async () => {
|
||
await fs.mkdir(path.dirname(baselinePath()), { recursive: true });
|
||
await writeFakePng(baselinePath());
|
||
};
|
||
|
||
it("removes the whole scratch dir on a within-tolerance pass", async () => {
|
||
await seedBaseline();
|
||
h.writeContextDiff = true; // the real differ writes both files even on a pass
|
||
|
||
const r = await runSnapshot(env, opts());
|
||
|
||
expect(r.status).toBe("pass");
|
||
expect(h.outputDir).not.toBe("");
|
||
await expect(fs.access(h.outputDir)).rejects.toThrow();
|
||
});
|
||
|
||
it("keeps only the registered context diff on failure", async () => {
|
||
await seedBaseline();
|
||
h.mismatchPercentage = 3.1;
|
||
h.writeContextDiff = true;
|
||
|
||
const r = await runSnapshot(env, opts());
|
||
|
||
expect(r.status).toBe("fail");
|
||
// The registered artifact's host path must survive for materialization…
|
||
await expect(fs.access(h.contextDiffPath)).resolves.toBeUndefined();
|
||
// …and it is the only leftover — the unregistered full-res diff is gone.
|
||
await expect(fs.readdir(h.outputDir)).resolves.toEqual([path.basename(h.contextDiffPath)]);
|
||
});
|
||
|
||
it("removes the scratch dir when a failure produced no context diff", async () => {
|
||
await seedBaseline();
|
||
h.mismatchPercentage = 100;
|
||
|
||
const r = await runSnapshot(env, opts());
|
||
|
||
expect(r.status).toBe("fail");
|
||
await expect(fs.access(h.outputDir)).rejects.toThrow();
|
||
});
|
||
|
||
it("removes the scratch dir on a dimension-mismatch bail", async () => {
|
||
await seedBaseline();
|
||
// Aspect ratios must genuinely differ for a full-screen bail (see above).
|
||
h.dimensionMismatch = {
|
||
expected: { width: 844, height: 390 },
|
||
actual: { width: 390, height: 844 },
|
||
};
|
||
|
||
const r = await runSnapshot(env, opts());
|
||
|
||
expect(r.status).toBe("fail");
|
||
await expect(fs.access(h.outputDir)).rejects.toThrow();
|
||
});
|
||
});
|
||
|
||
describe("runSnapshot cropOn", () => {
|
||
const cropOn = { text: "Header", loose: true };
|
||
// A cropOn key = full-capture dims + hash of the canonical selector identity
|
||
// ([text, textMatches, identifier, role, loose]) — recomputed here
|
||
// independently to pin the on-disk format.
|
||
const cropKey = `home__ios-100x200-crop-${createHash("sha256")
|
||
.update(JSON.stringify(["Header", null, null, null, true]))
|
||
.digest("hex")
|
||
.slice(0, 8)}`;
|
||
// 100×200 capture; the frame's pixel rect is x 25–75, y 50–100 → a 50×50 crop.
|
||
const frame = { x: 0.25, y: 0.25, width: 0.5, height: 0.25 };
|
||
const cropBaselinePath = () => path.join(tmpDir, "__baselines__", "checkout", `${cropKey}.png`);
|
||
|
||
beforeEach(async () => {
|
||
await writeRealPng(h.shotPath, 100, 200);
|
||
h.cropFrame = frame;
|
||
});
|
||
|
||
it("stores the cropped region as the baseline, keyed by the full capture", async () => {
|
||
vi.mocked(settleTree).mockClear();
|
||
|
||
const r = await runSnapshot(env, opts({ updateBaselines: true, cropOn }));
|
||
|
||
expect(r.status).toBe("pass");
|
||
// The third argument is the evidence sink the failure report reads from.
|
||
// `expect.anything()` matched any object at all, including the wrong one —
|
||
// assert the SHAPE `createTreeReadSink` produces, and that it was written.
|
||
expect(vi.mocked(waitForFrame)).toHaveBeenCalledWith(
|
||
env,
|
||
cropOn,
|
||
expect.objectContaining({ attempts: expect.any(Number) })
|
||
);
|
||
const sink = vi.mocked(waitForFrame).mock.calls[0]![2] as { attempts: number };
|
||
expect(sink.attempts).toBeGreaterThan(0);
|
||
// waitForFrame settles internally — the plain settle must not run too.
|
||
expect(vi.mocked(settleTree)).not.toHaveBeenCalled();
|
||
// Key: the FULL capture's dimensions (device-class identity) plus the
|
||
// selector hash (crop identity). Content: the crop.
|
||
expect(r.snapshotKey).toBe(cropKey);
|
||
await expect(pngSize(cropBaselinePath())).resolves.toEqual({ w: 50, h: 50 });
|
||
});
|
||
|
||
it("compares the cropped image and sweeps the crop scratch dir on a pass", async () => {
|
||
await fs.mkdir(path.dirname(cropBaselinePath()), { recursive: true });
|
||
await writeRealPng(cropBaselinePath(), 50, 50);
|
||
|
||
const r = await runSnapshot(env, opts({ cropOn }));
|
||
|
||
expect(r.status).toBe("pass");
|
||
// The differ compared the cropped scratch file, not the full capture…
|
||
expect(h.diffCurrentPath).not.toBe(h.shotPath);
|
||
expect(path.basename(h.diffCurrentPath)).toBe(`${cropKey}.png`);
|
||
// Crops are never top-masked — the top of a crop is element content, not
|
||
// the full screen's status bar.
|
||
expect(h.diffTopMask).toBe("none");
|
||
// Crop dims carry meaning — the differ must hard-fail any size drift.
|
||
expect(h.diffNormalizeSizes).toBe(false);
|
||
// …and the unregistered crop did not outlive the call.
|
||
await expect(fs.access(path.dirname(h.diffCurrentPath))).rejects.toThrow();
|
||
});
|
||
|
||
it("never masks a crop, even one overlapping the status-bar band", async () => {
|
||
// y 0.02–0.10 overlaps the top band — but masking a crop's overlap would
|
||
// degenerate into comparing NOTHING for an element fully inside the band
|
||
// (a vacuous pass), so a crop compares every pixel wherever it sits.
|
||
h.cropFrame = { x: 0, y: 0.02, width: 0.5, height: 0.08 };
|
||
await fs.mkdir(path.dirname(cropBaselinePath()), { recursive: true });
|
||
await writeRealPng(cropBaselinePath(), 50, 16);
|
||
|
||
const r = await runSnapshot(env, opts({ cropOn }));
|
||
|
||
expect(r.status).toBe("pass");
|
||
expect(h.diffTopMask).toBe("none");
|
||
});
|
||
|
||
it("crops exactly the frame's pixel rect from the capture", async () => {
|
||
await writeCoordPng(h.shotPath, 100, 200);
|
||
|
||
const r = await runSnapshot(env, opts({ updateBaselines: true, cropOn }));
|
||
|
||
expect(r.status).toBe("pass");
|
||
// frame {x: 0.25, y: 0.25, w: 0.5, h: 0.25} on 100×200 → rect x 25–75,
|
||
// y 50–100. The corner pixels' coordinate encoding pins the exact rect.
|
||
await expect(pngPixel(cropBaselinePath(), 0, 0)).resolves.toEqual([25, 50, 75]);
|
||
await expect(pngPixel(cropBaselinePath(), 49, 0)).resolves.toEqual([74, 50, 124]);
|
||
await expect(pngPixel(cropBaselinePath(), 0, 49)).resolves.toEqual([25, 99, 124]);
|
||
await expect(pngPixel(cropBaselinePath(), 49, 49)).resolves.toEqual([74, 99, 173]);
|
||
});
|
||
|
||
it("propagates a tree-source outage while resolving cropOn as an error", async () => {
|
||
// Deliberate asymmetry with the full-screen path's swallowed settle
|
||
// outage: without a tree there is no frame, and degrading to a full-screen
|
||
// capture would "compare" the whole screen against a cropped baseline.
|
||
// flow-run's snapshot arm turns the throw into a step error.
|
||
h.cropFrameError = new Error("native devtools disconnected");
|
||
vi.mocked(invokeOnDevice).mockClear();
|
||
|
||
await expect(runSnapshot(env, opts({ cropOn }))).rejects.toThrow(
|
||
"native devtools disconnected"
|
||
);
|
||
// Failed before capturing — no screenshot was taken.
|
||
expect(vi.mocked(invokeOnDevice)).not.toHaveBeenCalled();
|
||
});
|
||
|
||
it("returns the cropped image as `current` on a missing baseline", async () => {
|
||
const r = await runSnapshot(env, opts({ cropOn }));
|
||
|
||
expect(r.status).toBe("fail");
|
||
expect(r.reason).toContain('no baseline for "home"');
|
||
const current = r.artifacts?.current as { hostPath: string };
|
||
expect(current.hostPath).not.toBe(h.shotPath);
|
||
// The artifact is what would have been compared — the crop, kept alive
|
||
// past the scratch-dir sweep for later materialization.
|
||
await expect(pngSize(current.hostPath)).resolves.toEqual({ w: 50, h: 50 });
|
||
});
|
||
|
||
it("keeps only the registered cropped `current` on an over-threshold failure", async () => {
|
||
await fs.mkdir(path.dirname(cropBaselinePath()), { recursive: true });
|
||
await writeRealPng(cropBaselinePath(), 50, 50);
|
||
h.mismatchPercentage = 3.1;
|
||
|
||
const r = await runSnapshot(env, opts({ cropOn }));
|
||
|
||
expect(r.status).toBe("fail");
|
||
const current = r.artifacts?.current as { hostPath: string; filename: string };
|
||
await expect(pngSize(current.hostPath)).resolves.toEqual({ w: 50, h: 50 });
|
||
await expect(fs.readdir(path.dirname(current.hostPath))).resolves.toEqual([`${cropKey}.png`]);
|
||
// The crop file on disk shares the baseline's basename, and a remote client
|
||
// materializes downloads by filename — the two handles must not collide.
|
||
expect(current.filename).toBe(`${cropKey}-current.png`);
|
||
expect(r.artifacts?.baseline).toMatchObject({ filename: `${cropKey}.png` });
|
||
});
|
||
|
||
it("fails with the standard not-found reason without capturing when cropOn never resolves", async () => {
|
||
h.cropFrame = undefined;
|
||
vi.mocked(invokeOnDevice).mockClear();
|
||
|
||
const r = await runSnapshot(env, opts({ cropOn }));
|
||
|
||
expect(r.status).toBe("fail");
|
||
expect(r.reason).toContain('no visible element matched selector text="Header"');
|
||
expect(r.reason).toContain("scroll-to");
|
||
expect(vi.mocked(invokeOnDevice)).not.toHaveBeenCalled();
|
||
// With no readable tree behind it, a cropOn miss is an ENVIRONMENT failure:
|
||
// the selector was never actually looked for, so "not found" would send the
|
||
// author editing a selector nobody evaluated.
|
||
expect(r.evidence).toMatchObject({ code: "tree-source-unavailable", selector: cropOn });
|
||
});
|
||
|
||
it("classifies a cropOn miss against a READABLE tree as a selector failure", async () => {
|
||
// A cropOn miss is a SELECTOR failure, not a snapshot one — it earns the
|
||
// directives' own code and their candidates, derived from the tree the
|
||
// sink published.
|
||
h.cropFrame = undefined;
|
||
h.cropSinkTree = {
|
||
role: "AXWindow",
|
||
frame: { x: 0, y: 0, width: 1, height: 1 },
|
||
children: [
|
||
{
|
||
role: "AXStaticText",
|
||
label: "Heading",
|
||
frame: { x: 0.1, y: 0.1, width: 0.5, height: 0.05 },
|
||
children: [],
|
||
},
|
||
],
|
||
};
|
||
|
||
const r = await runSnapshot(env, opts({ cropOn }));
|
||
|
||
expect(r.status).toBe("fail");
|
||
expect(r.evidence).toMatchObject({ code: "selector-not-found", selector: cropOn });
|
||
expect(r.evidence?.matches).toEqual([]);
|
||
});
|
||
|
||
it("splits a cropOn miss into not-visible when the element has no frame", async () => {
|
||
// The same not-found/not-visible split every directive gets, driven from
|
||
// the tree the sink actually carries.
|
||
h.cropFrame = undefined;
|
||
h.cropSinkTree = {
|
||
role: "AXWindow",
|
||
frame: { x: 0, y: 0, width: 1, height: 1 },
|
||
children: [
|
||
{
|
||
role: "AXStaticText",
|
||
label: "Header",
|
||
frame: { x: 0.1, y: 0.1, width: 0, height: 0 },
|
||
children: [],
|
||
},
|
||
],
|
||
};
|
||
|
||
const r = await runSnapshot(env, opts({ cropOn }));
|
||
|
||
expect(r.status).toBe("fail");
|
||
expect(r.evidence).toMatchObject({ code: "selector-not-visible", selector: cropOn });
|
||
expect(r.evidence?.hint).toContain("zero area");
|
||
expect(r.evidence?.matches).toHaveLength(1);
|
||
});
|
||
|
||
it("skips without capturing when the run is aborted while resolving cropOn", async () => {
|
||
h.cropFrame = "aborted";
|
||
vi.mocked(invokeOnDevice).mockClear();
|
||
|
||
const r = await runSnapshot(env, opts({ cropOn }));
|
||
|
||
expect(r.status).toBe("skip");
|
||
expect(r.reason).toContain("aborted");
|
||
expect(vi.mocked(invokeOnDevice)).not.toHaveBeenCalled();
|
||
});
|
||
|
||
it("names element-size drift on a dimension-mismatch bail", async () => {
|
||
await fs.mkdir(path.dirname(cropBaselinePath()), { recursive: true });
|
||
await writeRealPng(cropBaselinePath(), 50, 60);
|
||
h.dimensionMismatch = {
|
||
expected: { width: 50, height: 60 },
|
||
actual: { width: 50, height: 50 },
|
||
};
|
||
|
||
const r = await runSnapshot(env, opts({ cropOn }));
|
||
|
||
expect(r.status).toBe("fail");
|
||
expect(r.reason).toContain("cropOn region is 50x50");
|
||
expect(r.reason).toContain("crop a fixed-size container");
|
||
// Same collision risk as the over-threshold path: both handles download.
|
||
expect(r.artifacts?.current).toMatchObject({ filename: `${cropKey}-current.png` });
|
||
expect(r.artifacts?.baseline).toMatchObject({ filename: `${cropKey}.png` });
|
||
});
|
||
|
||
it("fails a sub-pixel crop region instead of writing an empty PNG", async () => {
|
||
h.cropFrame = { x: 0.5, y: 0.5, width: 0.001, height: 0.001 };
|
||
// runSnapshot builds its crop scratch dir with
|
||
// mkdtemp(join(os.tmpdir(), "argent-flow-crop-")). Point the tmpdir at a
|
||
// dir only this test owns, so the leftover sweep below sees this run's crop
|
||
// dirs and nothing else — scanning the machine-wide tmpdir would also list
|
||
// the in-flight crop dir of any concurrent run.
|
||
const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "argent-flow-crop-scan-"));
|
||
const restoreTmpdir = redirectTmpdir(scratch);
|
||
|
||
try {
|
||
const r = await runSnapshot(env, opts({ cropOn }));
|
||
|
||
expect(r.status).toBe("fail");
|
||
expect(r.reason).toContain("empty at this resolution");
|
||
// The key still names the failure for an exporter (CLI --output), and the
|
||
// FULL capture is attached as `current` — no crop exists to show.
|
||
expect(r.snapshotKey).toBe(cropKey);
|
||
expect(r.artifacts?.current).toMatchObject({ hostPath: h.shotPath });
|
||
// The crop scratch dir (which never received a file) was swept.
|
||
const leftoverCropDirs = (await fs.readdir(scratch)).filter((e) =>
|
||
e.startsWith("argent-flow-crop-")
|
||
);
|
||
expect(leftoverCropDirs).toEqual([]);
|
||
// The selector DID resolve — that is how the crop got a frame — so the
|
||
// payload names it, and the assembler suppresses candidates on the code.
|
||
expect(r.evidence).toMatchObject({
|
||
code: "snapshot-crop-empty",
|
||
selector: cropOn,
|
||
expected: { kind: "snapshot", snapshotKey: cropKey },
|
||
});
|
||
} finally {
|
||
restoreTmpdir();
|
||
await fs.rm(scratch, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
it("keys same-name snapshots with different cropOn selectors to distinct baselines", async () => {
|
||
const r1 = await runSnapshot(env, opts({ updateBaselines: true, cropOn: { text: "Header" } }));
|
||
const r2 = await runSnapshot(
|
||
env,
|
||
opts({ updateBaselines: true, cropOn: { identifier: "hdr" } })
|
||
);
|
||
|
||
expect(r1.snapshotKey).toContain("-crop-");
|
||
expect(r2.snapshotKey).toContain("-crop-");
|
||
expect(r1.snapshotKey).not.toBe(r2.snapshotKey);
|
||
// Two baseline files on disk — the second write did not clobber the first.
|
||
const files = await fs.readdir(path.join(tmpDir, "__baselines__", "checkout"));
|
||
expect(files.sort()).toEqual([`${r1.snapshotKey}.png`, `${r2.snapshotKey}.png`].sort());
|
||
});
|
||
|
||
it("keys a selector canonically regardless of property insertion order", async () => {
|
||
const r1 = await runSnapshot(
|
||
env,
|
||
opts({ updateBaselines: true, cropOn: { text: "a", role: "b" } })
|
||
);
|
||
const r2 = await runSnapshot(
|
||
env,
|
||
opts({ updateBaselines: true, cropOn: { role: "b", text: "a" } })
|
||
);
|
||
|
||
expect(r1.snapshotKey).toBe(r2.snapshotKey);
|
||
});
|
||
|
||
it("keys loose and strict spellings of the same text differently", async () => {
|
||
// `loose` changes resolution (identifier-first fallback) — a different element.
|
||
const r1 = await runSnapshot(
|
||
env,
|
||
opts({ updateBaselines: true, cropOn: { text: "foo", loose: true } })
|
||
);
|
||
const r2 = await runSnapshot(env, opts({ updateBaselines: true, cropOn: { text: "foo" } }));
|
||
|
||
expect(r1.snapshotKey).not.toBe(r2.snapshotKey);
|
||
});
|
||
|
||
it("fails a crop key recaptured from another app, like any other key", async () => {
|
||
const seenKeys = new Map<string, string>();
|
||
|
||
const r1 = await runSnapshot(env, opts({ updateBaselines: true, cropOn, seenKeys }));
|
||
const r2 = await runSnapshot(
|
||
env,
|
||
opts({ updateBaselines: true, cropOn, seenKeys, appIdentity: "/apps/app-b" })
|
||
);
|
||
|
||
expect(r1.status).toBe("pass");
|
||
expect(r2.status).toBe("fail");
|
||
expect(r2.reason).toContain("already captured in this run from a different app (/apps/app-a)");
|
||
expect(r2.reason).toContain(`${cropKey}.png`);
|
||
// One file on disk: app-b never wrote over app-a's crop.
|
||
const files = await fs.readdir(path.join(tmpDir, "__baselines__", "checkout"));
|
||
expect(files).toEqual([`${cropKey}.png`]);
|
||
});
|
||
|
||
it("lets two apps share a snapshot name when they crop different elements", async () => {
|
||
// The keys carry the selector, so the two never share a baseline file —
|
||
// a guard keyed on the snapshot NAME would wrongly reject this.
|
||
const seenKeys = new Map<string, string>();
|
||
|
||
const r1 = await runSnapshot(
|
||
env,
|
||
opts({ updateBaselines: true, cropOn: { text: "Header" }, seenKeys })
|
||
);
|
||
const r2 = await runSnapshot(
|
||
env,
|
||
opts({
|
||
updateBaselines: true,
|
||
cropOn: { identifier: "hdr" },
|
||
seenKeys,
|
||
appIdentity: "/apps/app-b",
|
||
})
|
||
);
|
||
|
||
expect(r1.status).toBe("pass");
|
||
expect(r2.status).toBe("pass");
|
||
expect(r1.snapshotKey).not.toBe(r2.snapshotKey);
|
||
const files = await fs.readdir(path.join(tmpDir, "__baselines__", "checkout"));
|
||
expect(files.sort()).toEqual([`${r1.snapshotKey}.png`, `${r2.snapshotKey}.png`].sort());
|
||
});
|
||
});
|