Files
callstack__agent-device/packages/contracts/src/scroll-gesture.ts
Michał Pierzchała f843dc2df1 fix(scroll): keep saturated scroll gestures out of the status bar; gate Android replays from android/emulator (#1781 A1) (#1820)
* fix(scroll): keep saturated scroll gestures out of the status bar; gate Android replays from android/emulator (#1781 A1)

`pnpm gate replay-android` failed 4/8 whenever it ran after the full-tier Android E2E
(replays-nightly run 32107665052, job 95620294899): 05-app-lifecycle, 06-swipe-gestures and
both fixture replays diverged under "A system surface covers the app". The E2E was not the
cause. Reproduced on a pixel_7 / API 36 AVD with the same cutout geometry CI's
`avdmanager --device pixel_7` produces (status bar 136px, not the 63px of a plain 1080x2400
skin):

- `03-scroll-discovery.ad` runs `scroll up 3`. The scroll planner clamps travel to the viewport
  minus a 5% band, so the touch-down landed at y=120 — inside the 136px status bar — and
  pulled the notification shade instead of scrolling. On API 36 the app window is
  edge-to-edge, so the reported viewport starts at y=0 and includes that bar.
- The shade then covered every replay until `04`'s `back` closed it. Native readdir order on
  the runner (03, 05, 06, fixture/02, fixture/01, 04, 01, 02) put four files in that window;
  the last green run (2026-07-30) had 04 right after 03, so the pull was masked.

Fix in the product, not the lane: DEFAULT_EDGE_PADDING_FRACTION 0.05 -> 0.1 in the TS scroll
planner and its Swift port. Every real Pixel has a cutout (5.7% of a Pixel 7's height) and an
iPhone's Dynamic Island status bar is 6.9%, so any saturated `scroll up` opened the shade /
Notification Center for real agents too. Parity vectors updated in both suites plus a Pixel 7
regression vector (1080x2400, amount 3 -> touch-down y=240 > 136).

Second contamination the same order exposed once the shade was gone:
`fixture/02-selector-routes-covered-diagnosis.ad` is a #1715 reproduction recipe that FAILS
BY DESIGN at step 9 (covered-target refusal) and leaves the device in landscape, yet the
gate enumerated `test/integration/replays/android` recursively. iOS keeps gate replays in
`replays/ios/simulator` and fixture recipes in `replays/ios/fixture`; Android now mirrors that:
the six Settings replays move to `replays/android/emulator`, `test:replay:android` points there,
and `fixture/` stays E2E-owned (`full:fixture-replays` already runs 01 by path). android.yml
and the workflow-evidence fixture follow the path; the replay-compat manifest keeps the
historical paths it pins at released tags.

Verified live (Pixel 7 geometry, API 36, --retries 0): control run at main head in CI order
reproduces exactly CI's 4/8; with the fix, `pnpm gate replay-android` 6/6 in both native and
CI order, and `03` leaves Settings on screen (scroll up 3 now touches down at y=240).

* test(scroll): drive the TS and Swift scroll-plan parity vectors from one table (#1820 review)

The two suites hand-mirrored the same vectors and #1820 had to edit both by hand — the drift
class the repo already closes for the tap-point rule via contracts/fixtures/tap-point-policy.json.
The scroll vectors (plus both planner constants, pinned behaviourally on a 1000px axis) now live in
contracts/fixtures/scroll-gesture.json; scroll-gesture.test.ts and RunnerTests+ScrollGesture.swift
iterate it. Verified: vitest 10/10; the four XCTests run on an iOS 26.2 simulator with the unit
flag on (Executed 4 tests, 0 failures).

Also: test/ci/android-workflow-evidence.json says what it guards.

Follow-up for content-safe viewport bounds + discovery order: #1821.
2026-08-18 14:31:54 +02:00

326 lines
10 KiB
TypeScript

import { AppError } from '@agent-device/kernel/errors';
import { defineStringEnum } from './string-enum.ts';
import { isViewportRootNode } from './snapshot-visibility.ts';
import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot';
// What a caller may ASK for, as opposed to `ScrollDirection` (what the gesture resolves to):
// `top`/`bottom` are scroll-to-extreme requests with no direction of their own. Declared here
// rather than in the command runtime that resolves them, so the public `ScrollOptions` can be
// stated without depending on `commands/`.
export const SCROLL_INPUT_DIRECTIONS = ['up', 'down', 'left', 'right', 'top', 'bottom'] as const;
export type ScrollInputDirection = (typeof SCROLL_INPUT_DIRECTIONS)[number];
export const SCROLL_DIRECTIONS = ['up', 'down', 'left', 'right'] as const;
export type ScrollDirection = (typeof SCROLL_DIRECTIONS)[number];
export const SWIPE_PRESETS = ['left', 'right', 'left-edge', 'right-edge'] as const;
export type SwipePreset = (typeof SWIPE_PRESETS)[number];
export const SWIPE_PATTERNS = ['one-way', 'ping-pong'] as const;
export type SwipePattern = (typeof SWIPE_PATTERNS)[number];
export const SWIPE_REPETITION_MAX = 200;
export const SWIPE_PAUSE_MAX_MS = 10_000;
export const SWIPE_SERIES_MAX_SCHEDULED_DURATION_MS = 60_000;
const SCROLL_DIRECTION_ENUM = defineStringEnum(SCROLL_DIRECTIONS, {
message: (direction) => `Unknown direction: ${direction}`,
});
export type TransformGestureParams = {
x: number;
y: number;
dx: number;
dy: number;
scale: number;
degrees: number;
durationMs?: number;
};
export type GestureReferenceFrame = {
referenceWidth: number;
referenceHeight: number;
};
type GesturePoint = {
x: number;
y: number;
};
export type ScrollGestureOptions = {
direction: ScrollDirection;
amount?: number;
pixels?: number;
referenceWidth: number;
referenceHeight: number;
};
export type ScrollGesturePlan = {
direction: ScrollDirection;
x1: number;
y1: number;
x2: number;
y2: number;
referenceWidth: number;
referenceHeight: number;
amount?: number;
pixels: number;
};
export type SwipePresetGesturePlan = {
preset: SwipePreset;
x1: number;
y1: number;
x2: number;
y2: number;
referenceWidth: number;
referenceHeight: number;
};
export type InPageSwipeGesturePlan = {
direction: ScrollDirection;
x1: number;
y1: number;
x2: number;
y2: number;
referenceWidth: number;
referenceHeight: number;
};
const DEFAULT_SCROLL_AMOUNT = 0.6;
// Scroll gestures never touch the outer 10% of either axis. Modern app windows are edge-to-edge,
// so the viewport includes the system bars: a swipe that starts inside the status bar (5.7% of a
// Pixel 7's height, 6.9% of an iPhone's with a Dynamic Island) pulls the notification shade or
// Notification Center down instead of scrolling. 10% clears both with margin (#1781 A1).
const DEFAULT_EDGE_PADDING_FRACTION = 0.1;
// Edge presets stay close to the system gesture boundary without emitting edge coordinates.
const SWIPE_PRESET_EDGE_MARGIN_PX = 8;
export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGesturePlan {
const direction = options.direction;
const axisLength =
direction === 'up' || direction === 'down' ? options.referenceHeight : options.referenceWidth;
const requestedAmount = resolveRequestedAmount(options.amount);
const requestedPixels =
options.pixels !== undefined
? normalizeRequestedPixels(options.pixels)
: Math.round(axisLength * requestedAmount);
const edgePadding = Math.max(1, Math.round(axisLength * DEFAULT_EDGE_PADDING_FRACTION));
const maxTravelPixels = Math.max(1, axisLength - edgePadding * 2);
const travelPixels = Math.max(1, Math.min(requestedPixels, maxTravelPixels));
const halfTravel = Math.round(travelPixels / 2);
const centerX = Math.round(options.referenceWidth / 2);
const centerY = Math.round(options.referenceHeight / 2);
const buildPlan = (x1: number, y1: number, x2: number, y2: number): ScrollGesturePlan => ({
direction,
x1,
y1,
x2,
y2,
referenceWidth: options.referenceWidth,
referenceHeight: options.referenceHeight,
amount: options.amount,
pixels: travelPixels,
});
switch (direction) {
case 'up':
return buildPlan(centerX, centerY - halfTravel, centerX, centerY + halfTravel);
case 'down':
return buildPlan(centerX, centerY + halfTravel, centerX, centerY - halfTravel);
case 'left':
return buildPlan(centerX - halfTravel, centerY, centerX + halfTravel, centerY);
case 'right':
return buildPlan(centerX + halfTravel, centerY, centerX - halfTravel, centerY);
}
}
/**
* Validates pre-frame scroll inputs (amount/pixels) the same way buildScrollGesturePlan would,
* so the daemon throws INVALID_ARGS for bad inputs BEFORE sending the fused runner `scroll`
* command (previously validation ran between the frame request and the drag). The resolved
* values are discarded; only their throw-on-invalid behavior is reused.
*/
export function assertScrollGestureInput(options: { amount?: number; pixels?: number }): void {
resolveRequestedAmount(options.amount);
if (options.pixels !== undefined) {
normalizeRequestedPixels(options.pixels);
}
}
export function buildSwipePresetGesturePlan(
preset: SwipePreset,
frame: GestureReferenceFrame,
): SwipePresetGesturePlan {
if (preset === 'left' || preset === 'right') {
const plan = buildInPageSwipeGesturePlan(preset, frame);
return {
preset,
x1: plan.x1,
y1: plan.y1,
x2: plan.x2,
y2: plan.y2,
referenceWidth: plan.referenceWidth,
referenceHeight: plan.referenceHeight,
};
}
const [startPercent, endPercent] = preset === 'left-edge' ? [99, 15] : [1, 85];
const start = clampGesturePoint(
pointFromPercent(frame, startPercent, 50),
frame,
SWIPE_PRESET_EDGE_MARGIN_PX,
);
const end = clampGesturePoint(
pointFromPercent(frame, endPercent, 50),
frame,
SWIPE_PRESET_EDGE_MARGIN_PX,
);
return {
preset,
x1: start.x,
y1: start.y,
x2: end.x,
y2: end.y,
referenceWidth: frame.referenceWidth,
referenceHeight: frame.referenceHeight,
};
}
export function gestureDirectionDelta(direction: ScrollDirection, distance: number): GesturePoint {
switch (direction) {
case 'up':
return { x: 0, y: -distance };
case 'down':
return { x: 0, y: distance };
case 'left':
return { x: -distance, y: 0 };
case 'right':
return { x: distance, y: 0 };
}
}
/** Plans a centered, edge-inset finger motion that remains inside app content. */
export function buildInPageSwipeGesturePlan(
direction: ScrollDirection,
frame: GestureReferenceFrame,
): InPageSwipeGesturePlan {
// These insets avoid system-edge gestures while retaining enough travel for pagers.
const startPercent = 85;
const endPercent = 15;
const centerPercent = 50;
const forward = direction === 'left' || direction === 'up';
const startAxisPercent = forward ? startPercent : endPercent;
const endAxisPercent = forward ? endPercent : startPercent;
const vertical = direction === 'up' || direction === 'down';
const start = clampGesturePoint(
pointFromPercent(
frame,
vertical ? centerPercent : startAxisPercent,
vertical ? startAxisPercent : centerPercent,
),
frame,
SWIPE_PRESET_EDGE_MARGIN_PX,
);
const end = clampGesturePoint(
pointFromPercent(
frame,
vertical ? centerPercent : endAxisPercent,
vertical ? endAxisPercent : centerPercent,
),
frame,
SWIPE_PRESET_EDGE_MARGIN_PX,
);
return {
direction,
x1: start.x,
y1: start.y,
x2: end.x,
y2: end.y,
referenceWidth: frame.referenceWidth,
referenceHeight: frame.referenceHeight,
};
}
export function inferGestureReferenceFrame(
nodes: Array<Pick<SnapshotNode, 'type' | 'rect'>>,
): GestureReferenceFrame | undefined {
const viewportRect = inferViewportRect(nodes);
if (!viewportRect) return undefined;
return {
referenceWidth: viewportRect.width,
referenceHeight: viewportRect.height,
};
}
function pointFromPercent(
frame: GestureReferenceFrame,
xPercent: number,
yPercent: number,
): GesturePoint {
const x = Math.trunc((frame.referenceWidth * xPercent) / 100);
const y = Math.trunc((frame.referenceHeight * yPercent) / 100);
return {
x: Number.isFinite(x) ? x : 0,
y: Number.isFinite(y) ? y : 0,
};
}
function clampGesturePoint(
point: GesturePoint,
frame: GestureReferenceFrame,
marginPx: number,
): GesturePoint {
return {
x: clampGestureCoordinate(point.x, marginPx, frame.referenceWidth),
y: clampGestureCoordinate(point.y, marginPx, frame.referenceHeight),
};
}
export function parseScrollDirection(direction: string): ScrollDirection {
return SCROLL_DIRECTION_ENUM.parse(direction);
}
function inferViewportRect(nodes: Array<Pick<SnapshotNode, 'type' | 'rect'>>): Rect | undefined {
const candidate = nodes
.filter((node) => isViewportRootNode(node) && isValidRect(node.rect))
.map((node) => node.rect)
.sort(
(left, right) =>
(right?.width ?? 0) * (right?.height ?? 0) - (left?.width ?? 0) * (left?.height ?? 0),
)[0];
if (candidate) return candidate;
const rects = nodes.map((node) => node.rect).filter(isValidRect);
if (rects.length === 0) return undefined;
const width = Math.max(...rects.map((rect) => rect.x + rect.width));
const height = Math.max(...rects.map((rect) => rect.y + rect.height));
if (width <= 0 || height <= 0) return undefined;
return { x: 0, y: 0, width, height };
}
function isValidRect(rect: Rect | undefined): rect is Rect {
return !!rect && rect.width > 0 && rect.height > 0;
}
function resolveRequestedAmount(amount: number | undefined): number {
if (amount === undefined) return DEFAULT_SCROLL_AMOUNT;
if (!Number.isFinite(amount) || amount <= 0) {
throw new AppError('INVALID_ARGS', 'scroll amount must be a positive number');
}
return amount;
}
function normalizeRequestedPixels(pixels: number): number {
if (!Number.isFinite(pixels) || pixels <= 0) {
throw new AppError('INVALID_ARGS', 'scroll pixels must be a positive integer');
}
return Math.max(1, Math.round(pixels));
}
export function clampGestureCoordinate(value: number, marginPx: number, size: number): number {
const min = Math.round(marginPx);
if (!Number.isFinite(min)) return 0;
const max = Math.max(min, Math.round(size - marginPx));
if (!Number.isFinite(max) || !Number.isFinite(value)) return min;
return Math.min(max, Math.max(min, Math.round(value)));
}