mirror of
https://github.com/callstack/agent-device.git
synced 2026-09-14 20:06:34 +08:00
f843dc2df1
* fix(scroll): keep saturated scroll gestures out of the status bar; gate Android replays from android/emulator (#1781 A1) `pnpm gate replay-android` failed 4/8 whenever it ran after the full-tier Android E2E (replays-nightly run 32107665052, job 95620294899): 05-app-lifecycle, 06-swipe-gestures and both fixture replays diverged under "A system surface covers the app". The E2E was not the cause. Reproduced on a pixel_7 / API 36 AVD with the same cutout geometry CI's `avdmanager --device pixel_7` produces (status bar 136px, not the 63px of a plain 1080x2400 skin): - `03-scroll-discovery.ad` runs `scroll up 3`. The scroll planner clamps travel to the viewport minus a 5% band, so the touch-down landed at y=120 — inside the 136px status bar — and pulled the notification shade instead of scrolling. On API 36 the app window is edge-to-edge, so the reported viewport starts at y=0 and includes that bar. - The shade then covered every replay until `04`'s `back` closed it. Native readdir order on the runner (03, 05, 06, fixture/02, fixture/01, 04, 01, 02) put four files in that window; the last green run (2026-07-30) had 04 right after 03, so the pull was masked. Fix in the product, not the lane: DEFAULT_EDGE_PADDING_FRACTION 0.05 -> 0.1 in the TS scroll planner and its Swift port. Every real Pixel has a cutout (5.7% of a Pixel 7's height) and an iPhone's Dynamic Island status bar is 6.9%, so any saturated `scroll up` opened the shade / Notification Center for real agents too. Parity vectors updated in both suites plus a Pixel 7 regression vector (1080x2400, amount 3 -> touch-down y=240 > 136). Second contamination the same order exposed once the shade was gone: `fixture/02-selector-routes-covered-diagnosis.ad` is a #1715 reproduction recipe that FAILS BY DESIGN at step 9 (covered-target refusal) and leaves the device in landscape, yet the gate enumerated `test/integration/replays/android` recursively. iOS keeps gate replays in `replays/ios/simulator` and fixture recipes in `replays/ios/fixture`; Android now mirrors that: the six Settings replays move to `replays/android/emulator`, `test:replay:android` points there, and `fixture/` stays E2E-owned (`full:fixture-replays` already runs 01 by path). android.yml and the workflow-evidence fixture follow the path; the replay-compat manifest keeps the historical paths it pins at released tags. Verified live (Pixel 7 geometry, API 36, --retries 0): control run at main head in CI order reproduces exactly CI's 4/8; with the fix, `pnpm gate replay-android` 6/6 in both native and CI order, and `03` leaves Settings on screen (scroll up 3 now touches down at y=240). * test(scroll): drive the TS and Swift scroll-plan parity vectors from one table (#1820 review) The two suites hand-mirrored the same vectors and #1820 had to edit both by hand — the drift class the repo already closes for the tap-point rule via contracts/fixtures/tap-point-policy.json. The scroll vectors (plus both planner constants, pinned behaviourally on a 1000px axis) now live in contracts/fixtures/scroll-gesture.json; scroll-gesture.test.ts and RunnerTests+ScrollGesture.swift iterate it. Verified: vitest 10/10; the four XCTests run on an iOS 26.2 simulator with the unit flag on (Executed 4 tests, 0 failures). Also: test/ci/android-workflow-evidence.json says what it guards. Follow-up for content-safe viewport bounds + discovery order: #1821.
326 lines
10 KiB
TypeScript
326 lines
10 KiB
TypeScript
import { AppError } from '@agent-device/kernel/errors';
|
|
import { defineStringEnum } from './string-enum.ts';
|
|
import { isViewportRootNode } from './snapshot-visibility.ts';
|
|
import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot';
|
|
|
|
// What a caller may ASK for, as opposed to `ScrollDirection` (what the gesture resolves to):
|
|
// `top`/`bottom` are scroll-to-extreme requests with no direction of their own. Declared here
|
|
// rather than in the command runtime that resolves them, so the public `ScrollOptions` can be
|
|
// stated without depending on `commands/`.
|
|
export const SCROLL_INPUT_DIRECTIONS = ['up', 'down', 'left', 'right', 'top', 'bottom'] as const;
|
|
export type ScrollInputDirection = (typeof SCROLL_INPUT_DIRECTIONS)[number];
|
|
|
|
export const SCROLL_DIRECTIONS = ['up', 'down', 'left', 'right'] as const;
|
|
export type ScrollDirection = (typeof SCROLL_DIRECTIONS)[number];
|
|
export const SWIPE_PRESETS = ['left', 'right', 'left-edge', 'right-edge'] as const;
|
|
export type SwipePreset = (typeof SWIPE_PRESETS)[number];
|
|
export const SWIPE_PATTERNS = ['one-way', 'ping-pong'] as const;
|
|
export type SwipePattern = (typeof SWIPE_PATTERNS)[number];
|
|
export const SWIPE_REPETITION_MAX = 200;
|
|
export const SWIPE_PAUSE_MAX_MS = 10_000;
|
|
export const SWIPE_SERIES_MAX_SCHEDULED_DURATION_MS = 60_000;
|
|
const SCROLL_DIRECTION_ENUM = defineStringEnum(SCROLL_DIRECTIONS, {
|
|
message: (direction) => `Unknown direction: ${direction}`,
|
|
});
|
|
|
|
export type TransformGestureParams = {
|
|
x: number;
|
|
y: number;
|
|
dx: number;
|
|
dy: number;
|
|
scale: number;
|
|
degrees: number;
|
|
durationMs?: number;
|
|
};
|
|
|
|
export type GestureReferenceFrame = {
|
|
referenceWidth: number;
|
|
referenceHeight: number;
|
|
};
|
|
|
|
type GesturePoint = {
|
|
x: number;
|
|
y: number;
|
|
};
|
|
|
|
export type ScrollGestureOptions = {
|
|
direction: ScrollDirection;
|
|
amount?: number;
|
|
pixels?: number;
|
|
referenceWidth: number;
|
|
referenceHeight: number;
|
|
};
|
|
|
|
export type ScrollGesturePlan = {
|
|
direction: ScrollDirection;
|
|
x1: number;
|
|
y1: number;
|
|
x2: number;
|
|
y2: number;
|
|
referenceWidth: number;
|
|
referenceHeight: number;
|
|
amount?: number;
|
|
pixels: number;
|
|
};
|
|
|
|
export type SwipePresetGesturePlan = {
|
|
preset: SwipePreset;
|
|
x1: number;
|
|
y1: number;
|
|
x2: number;
|
|
y2: number;
|
|
referenceWidth: number;
|
|
referenceHeight: number;
|
|
};
|
|
|
|
export type InPageSwipeGesturePlan = {
|
|
direction: ScrollDirection;
|
|
x1: number;
|
|
y1: number;
|
|
x2: number;
|
|
y2: number;
|
|
referenceWidth: number;
|
|
referenceHeight: number;
|
|
};
|
|
|
|
const DEFAULT_SCROLL_AMOUNT = 0.6;
|
|
// Scroll gestures never touch the outer 10% of either axis. Modern app windows are edge-to-edge,
|
|
// so the viewport includes the system bars: a swipe that starts inside the status bar (5.7% of a
|
|
// Pixel 7's height, 6.9% of an iPhone's with a Dynamic Island) pulls the notification shade or
|
|
// Notification Center down instead of scrolling. 10% clears both with margin (#1781 A1).
|
|
const DEFAULT_EDGE_PADDING_FRACTION = 0.1;
|
|
// Edge presets stay close to the system gesture boundary without emitting edge coordinates.
|
|
const SWIPE_PRESET_EDGE_MARGIN_PX = 8;
|
|
|
|
export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGesturePlan {
|
|
const direction = options.direction;
|
|
const axisLength =
|
|
direction === 'up' || direction === 'down' ? options.referenceHeight : options.referenceWidth;
|
|
const requestedAmount = resolveRequestedAmount(options.amount);
|
|
const requestedPixels =
|
|
options.pixels !== undefined
|
|
? normalizeRequestedPixels(options.pixels)
|
|
: Math.round(axisLength * requestedAmount);
|
|
const edgePadding = Math.max(1, Math.round(axisLength * DEFAULT_EDGE_PADDING_FRACTION));
|
|
const maxTravelPixels = Math.max(1, axisLength - edgePadding * 2);
|
|
const travelPixels = Math.max(1, Math.min(requestedPixels, maxTravelPixels));
|
|
const halfTravel = Math.round(travelPixels / 2);
|
|
const centerX = Math.round(options.referenceWidth / 2);
|
|
const centerY = Math.round(options.referenceHeight / 2);
|
|
const buildPlan = (x1: number, y1: number, x2: number, y2: number): ScrollGesturePlan => ({
|
|
direction,
|
|
x1,
|
|
y1,
|
|
x2,
|
|
y2,
|
|
referenceWidth: options.referenceWidth,
|
|
referenceHeight: options.referenceHeight,
|
|
amount: options.amount,
|
|
pixels: travelPixels,
|
|
});
|
|
|
|
switch (direction) {
|
|
case 'up':
|
|
return buildPlan(centerX, centerY - halfTravel, centerX, centerY + halfTravel);
|
|
case 'down':
|
|
return buildPlan(centerX, centerY + halfTravel, centerX, centerY - halfTravel);
|
|
case 'left':
|
|
return buildPlan(centerX - halfTravel, centerY, centerX + halfTravel, centerY);
|
|
case 'right':
|
|
return buildPlan(centerX + halfTravel, centerY, centerX - halfTravel, centerY);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Validates pre-frame scroll inputs (amount/pixels) the same way buildScrollGesturePlan would,
|
|
* so the daemon throws INVALID_ARGS for bad inputs BEFORE sending the fused runner `scroll`
|
|
* command (previously validation ran between the frame request and the drag). The resolved
|
|
* values are discarded; only their throw-on-invalid behavior is reused.
|
|
*/
|
|
export function assertScrollGestureInput(options: { amount?: number; pixels?: number }): void {
|
|
resolveRequestedAmount(options.amount);
|
|
if (options.pixels !== undefined) {
|
|
normalizeRequestedPixels(options.pixels);
|
|
}
|
|
}
|
|
|
|
export function buildSwipePresetGesturePlan(
|
|
preset: SwipePreset,
|
|
frame: GestureReferenceFrame,
|
|
): SwipePresetGesturePlan {
|
|
if (preset === 'left' || preset === 'right') {
|
|
const plan = buildInPageSwipeGesturePlan(preset, frame);
|
|
return {
|
|
preset,
|
|
x1: plan.x1,
|
|
y1: plan.y1,
|
|
x2: plan.x2,
|
|
y2: plan.y2,
|
|
referenceWidth: plan.referenceWidth,
|
|
referenceHeight: plan.referenceHeight,
|
|
};
|
|
}
|
|
const [startPercent, endPercent] = preset === 'left-edge' ? [99, 15] : [1, 85];
|
|
const start = clampGesturePoint(
|
|
pointFromPercent(frame, startPercent, 50),
|
|
frame,
|
|
SWIPE_PRESET_EDGE_MARGIN_PX,
|
|
);
|
|
const end = clampGesturePoint(
|
|
pointFromPercent(frame, endPercent, 50),
|
|
frame,
|
|
SWIPE_PRESET_EDGE_MARGIN_PX,
|
|
);
|
|
return {
|
|
preset,
|
|
x1: start.x,
|
|
y1: start.y,
|
|
x2: end.x,
|
|
y2: end.y,
|
|
referenceWidth: frame.referenceWidth,
|
|
referenceHeight: frame.referenceHeight,
|
|
};
|
|
}
|
|
|
|
export function gestureDirectionDelta(direction: ScrollDirection, distance: number): GesturePoint {
|
|
switch (direction) {
|
|
case 'up':
|
|
return { x: 0, y: -distance };
|
|
case 'down':
|
|
return { x: 0, y: distance };
|
|
case 'left':
|
|
return { x: -distance, y: 0 };
|
|
case 'right':
|
|
return { x: distance, y: 0 };
|
|
}
|
|
}
|
|
|
|
/** Plans a centered, edge-inset finger motion that remains inside app content. */
|
|
export function buildInPageSwipeGesturePlan(
|
|
direction: ScrollDirection,
|
|
frame: GestureReferenceFrame,
|
|
): InPageSwipeGesturePlan {
|
|
// These insets avoid system-edge gestures while retaining enough travel for pagers.
|
|
const startPercent = 85;
|
|
const endPercent = 15;
|
|
const centerPercent = 50;
|
|
const forward = direction === 'left' || direction === 'up';
|
|
const startAxisPercent = forward ? startPercent : endPercent;
|
|
const endAxisPercent = forward ? endPercent : startPercent;
|
|
const vertical = direction === 'up' || direction === 'down';
|
|
const start = clampGesturePoint(
|
|
pointFromPercent(
|
|
frame,
|
|
vertical ? centerPercent : startAxisPercent,
|
|
vertical ? startAxisPercent : centerPercent,
|
|
),
|
|
frame,
|
|
SWIPE_PRESET_EDGE_MARGIN_PX,
|
|
);
|
|
const end = clampGesturePoint(
|
|
pointFromPercent(
|
|
frame,
|
|
vertical ? centerPercent : endAxisPercent,
|
|
vertical ? endAxisPercent : centerPercent,
|
|
),
|
|
frame,
|
|
SWIPE_PRESET_EDGE_MARGIN_PX,
|
|
);
|
|
return {
|
|
direction,
|
|
x1: start.x,
|
|
y1: start.y,
|
|
x2: end.x,
|
|
y2: end.y,
|
|
referenceWidth: frame.referenceWidth,
|
|
referenceHeight: frame.referenceHeight,
|
|
};
|
|
}
|
|
|
|
export function inferGestureReferenceFrame(
|
|
nodes: Array<Pick<SnapshotNode, 'type' | 'rect'>>,
|
|
): GestureReferenceFrame | undefined {
|
|
const viewportRect = inferViewportRect(nodes);
|
|
if (!viewportRect) return undefined;
|
|
return {
|
|
referenceWidth: viewportRect.width,
|
|
referenceHeight: viewportRect.height,
|
|
};
|
|
}
|
|
|
|
function pointFromPercent(
|
|
frame: GestureReferenceFrame,
|
|
xPercent: number,
|
|
yPercent: number,
|
|
): GesturePoint {
|
|
const x = Math.trunc((frame.referenceWidth * xPercent) / 100);
|
|
const y = Math.trunc((frame.referenceHeight * yPercent) / 100);
|
|
return {
|
|
x: Number.isFinite(x) ? x : 0,
|
|
y: Number.isFinite(y) ? y : 0,
|
|
};
|
|
}
|
|
|
|
function clampGesturePoint(
|
|
point: GesturePoint,
|
|
frame: GestureReferenceFrame,
|
|
marginPx: number,
|
|
): GesturePoint {
|
|
return {
|
|
x: clampGestureCoordinate(point.x, marginPx, frame.referenceWidth),
|
|
y: clampGestureCoordinate(point.y, marginPx, frame.referenceHeight),
|
|
};
|
|
}
|
|
|
|
export function parseScrollDirection(direction: string): ScrollDirection {
|
|
return SCROLL_DIRECTION_ENUM.parse(direction);
|
|
}
|
|
|
|
function inferViewportRect(nodes: Array<Pick<SnapshotNode, 'type' | 'rect'>>): Rect | undefined {
|
|
const candidate = nodes
|
|
.filter((node) => isViewportRootNode(node) && isValidRect(node.rect))
|
|
.map((node) => node.rect)
|
|
.sort(
|
|
(left, right) =>
|
|
(right?.width ?? 0) * (right?.height ?? 0) - (left?.width ?? 0) * (left?.height ?? 0),
|
|
)[0];
|
|
if (candidate) return candidate;
|
|
|
|
const rects = nodes.map((node) => node.rect).filter(isValidRect);
|
|
if (rects.length === 0) return undefined;
|
|
|
|
const width = Math.max(...rects.map((rect) => rect.x + rect.width));
|
|
const height = Math.max(...rects.map((rect) => rect.y + rect.height));
|
|
if (width <= 0 || height <= 0) return undefined;
|
|
return { x: 0, y: 0, width, height };
|
|
}
|
|
|
|
function isValidRect(rect: Rect | undefined): rect is Rect {
|
|
return !!rect && rect.width > 0 && rect.height > 0;
|
|
}
|
|
|
|
function resolveRequestedAmount(amount: number | undefined): number {
|
|
if (amount === undefined) return DEFAULT_SCROLL_AMOUNT;
|
|
if (!Number.isFinite(amount) || amount <= 0) {
|
|
throw new AppError('INVALID_ARGS', 'scroll amount must be a positive number');
|
|
}
|
|
return amount;
|
|
}
|
|
|
|
function normalizeRequestedPixels(pixels: number): number {
|
|
if (!Number.isFinite(pixels) || pixels <= 0) {
|
|
throw new AppError('INVALID_ARGS', 'scroll pixels must be a positive integer');
|
|
}
|
|
return Math.max(1, Math.round(pixels));
|
|
}
|
|
|
|
export function clampGestureCoordinate(value: number, marginPx: number, size: number): number {
|
|
const min = Math.round(marginPx);
|
|
if (!Number.isFinite(min)) return 0;
|
|
|
|
const max = Math.max(min, Math.round(size - marginPx));
|
|
if (!Number.isFinite(max) || !Number.isFinite(value)) return min;
|
|
|
|
return Math.min(max, Math.max(min, Math.round(value)));
|
|
}
|