mirror of
https://github.com/software-mansion/argent.git
synced 2026-09-14 19:27:14 +08:00
b8afbed183
Adds end-to-end support for driving **Apple TV (tvOS)** and **Android TV (leanback)** targets through argent, using a focus-driven interaction model (TVs have no touchscreen — interaction is moving a focus highlight with the remote). This PR was reworked after review to **reuse the existing cross-platform tool surface and the `tv-remote` tool from the Vega/Fire TV work (#366)** rather than ship a separate family of `tv-*` tools. ## What this delivers TV targets are driven by the **same cross-platform tools as every other platform**, routed through the existing `dispatchByPlatform` fork mechanism — there are no dedicated `tv-describe`/`tv-navigate`/`tv-type`/`tv-set-focus` tools: - **`describe`** — on a `runtimeKind: "tv"` target, returns the focus-driven view (focused + focusable elements) instead of a tap tree. On Android TV it auto-falls-back to the full `uiautomator` tree when the RN focus engine exposes nothing. - **`tv-remote`** — the cross-platform remote/D-pad tool from #366, now extended with `ios` (Apple TV HID daemon) and `android` (Android TV adb keyevents) branches alongside the existing `vega` branch. Single button, a `repeat`, or a whole path (`["up","right","select"]`) in one call. - **`keyboard`** — types into the focused field on a TV target (named keys are rejected there — they're navigation, which belongs to `tv-remote`). - **`button`** stays hardware-only (phones/tablets); it is not a TV tool. **Apple TV** runs two native daemons (in-sim AX service + host-side HID daemon), shipped via the `argent-private` submodule. **Android TV** reuses the same tool surface, adb-backed (`input keyevent`, `uiautomator dump`, `input text`). ## Remote vocabulary parity `tv-remote` exposes the full 16-button vocabulary (`up`/`down`/`left`/`right`/`select`/`back`/`home`/`menu`/`playPause`/`rewind`/`fastForward`/`next`/`previous`/`volumeUp`/`volumeDown`/`mute`): | | Apple TV | Android TV | Vega | |---|---|---|---| | D-pad / select / back / menu / home / playPause | ✅ | ✅ | ✅ | | media-transport + volume/mute | ❌ rejected | ✅ | ✅ | Media-transport/volume keys **genuinely work on Android TV** (real keycodes — verified live: `volumeUp` moved `STREAM_MUSIC`, `mute` toggled per `dumpsys audio`). On the **Apple TV simulator** they are **rejected with a clear error**: on-device testing confirmed the tvOS sim's HID stack silently drops Consumer-Control events, so returning success would be a lie. ## Key design points - **tvOS UDIDs are UUID-shaped and indistinguishable from iOS by shape.** Handled with `runtimeKind` detection (tvOS runtime string for Apple TV; `pm list features` leanback/television for Android TV — *not* `ro.build.characteristics`, which lies on TV emulators). - Converted launch-app / restart-app / screenshot / screenshot-diff / run-sequence from **eager** to **lazy** service resolution, so a tvOS target never spins up (and hangs on) the iOS-only simulator-server / native-devtools blueprints. - Native injection selects the platform-matched (TVOSSIMULATOR) dylib slice; daemons recycle across sim reboots and app relaunches so injection and focus survive. - SKILL.md footprint minimized per review: the two TV skills were collapsed into a single lean **`argent-tv-interact`** (~40 lines). ## Verification - `npm run build`, ESLint, Prettier, and the full tool-server suite (**1565 tests**) all pass; CI green (incl. the Apple TV / Android TV / Vega e2e jobs). - Core flows verified live against real apps on both an Apple TV 4K simulator and a Google ATV emulator. ## Dependency / merge order Pins the `argent-private` submodule to the tip of its `feat/tv-support` branch (software-mansion/argent-private#20). **Merge that PR to `main` first, then bump the submodule pointer here to the merged SHA before merging this PR.** 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
149 lines
5.5 KiB
TypeScript
149 lines
5.5 KiB
TypeScript
import { describe, it, expect, vi } from "vitest";
|
|
import type { DeviceInfo } from "@argent/registry";
|
|
|
|
// The Android empty-focus fallback shells out to uiautomator via describeAndroid;
|
|
// stub it so the TV-describe routing can be tested without adb.
|
|
const describeAndroidMock = vi.fn();
|
|
vi.mock("../src/tools/describe/platforms/android", () => ({
|
|
describeAndroid: (...a: unknown[]) => describeAndroidMock(...a),
|
|
}));
|
|
|
|
import { describeTv } from "../src/tools/describe/platforms/tv";
|
|
import type { TvControlApi, TvDescribeResponse } from "../src/blueprints/tv-control";
|
|
|
|
// The tool touches `describe()` and `recycleAx()`; the rest is unused here.
|
|
function makeApi(
|
|
describeFn: TvControlApi["describe"],
|
|
recycleAx: TvControlApi["recycleAx"] = vi.fn().mockResolvedValue(undefined)
|
|
): TvControlApi {
|
|
return {
|
|
describe: describeFn,
|
|
recycleAx,
|
|
navigate: vi.fn(),
|
|
type: vi.fn(),
|
|
} as unknown as TvControlApi;
|
|
}
|
|
|
|
// describeTv resolves the TvControlApi through the registry; the unit tests
|
|
// inject it directly by stubbing resolveService.
|
|
function makeRegistry(api: TvControlApi) {
|
|
return { resolveService: vi.fn(async () => api) } as never;
|
|
}
|
|
|
|
// Apple TV target — platform "ios" by UDID shape, so no Android fallback fires.
|
|
const TVOS_DEVICE: DeviceInfo = {
|
|
id: "DDDDDDDD-DDDD-DDDD-DDDD-DDDDDDDDDDDD",
|
|
platform: "ios",
|
|
kind: "simulator",
|
|
};
|
|
|
|
const populated: TvDescribeResponse = {
|
|
bundleId: "com.nfl.gamecenter",
|
|
focused: { label: "Home", isFocused: true },
|
|
focusable: [{ label: "Home", isFocused: true }, { label: "Games" }],
|
|
};
|
|
|
|
const empty: TvDescribeResponse = {
|
|
bundleId: "com.nfl.gamecenter",
|
|
focused: null,
|
|
focusable: [],
|
|
};
|
|
|
|
async function run(api: TvControlApi) {
|
|
return describeTv(makeRegistry(api), TVOS_DEVICE);
|
|
}
|
|
|
|
describe("describe (TV) — empty-state resilience", () => {
|
|
it("returns the populated focus view without retrying", async () => {
|
|
const describeFn = vi.fn().mockResolvedValue(populated);
|
|
const res = await run(makeApi(describeFn));
|
|
|
|
expect(describeFn).toHaveBeenCalledTimes(1);
|
|
expect(res.source).toBe("tv-focus");
|
|
expect(res.description).toContain("Focused: Home");
|
|
expect(res.description).toContain("Focusable (2):");
|
|
expect(res.hint).toBeUndefined();
|
|
expect(res.description).not.toContain("Note:");
|
|
});
|
|
|
|
it("retries while empty and returns the tree once it populates (app finished loading)", async () => {
|
|
// First two probes hit the splash/loading window, third sees the real UI.
|
|
const describeFn = vi
|
|
.fn()
|
|
.mockResolvedValueOnce(empty)
|
|
.mockResolvedValueOnce(empty)
|
|
.mockResolvedValue(populated);
|
|
const res = await run(makeApi(describeFn));
|
|
|
|
expect(describeFn).toHaveBeenCalledTimes(3);
|
|
expect(res.description).toContain("Focusable (2):");
|
|
expect(res.hint).toBeUndefined();
|
|
});
|
|
|
|
it("recycles the daemon and recovers when the cache was stale", async () => {
|
|
// The transition-window retries stay empty (stale primaryApp cache), then a
|
|
// recycle rebinds to the real foreground app and the next probe populates.
|
|
const describeFn = vi
|
|
.fn()
|
|
.mockResolvedValueOnce(empty)
|
|
.mockResolvedValueOnce(empty)
|
|
.mockResolvedValueOnce(empty)
|
|
.mockResolvedValue(populated);
|
|
const recycleAx = vi.fn().mockResolvedValue(undefined);
|
|
const res = await run(makeApi(describeFn, recycleAx));
|
|
|
|
expect(describeFn).toHaveBeenCalledTimes(4);
|
|
expect(recycleAx).toHaveBeenCalledTimes(1);
|
|
expect(res.description).toContain("Focusable (2):");
|
|
expect(res.hint).toBeUndefined();
|
|
});
|
|
|
|
it("exhausts retries, recycles, and surfaces the hint when still empty", async () => {
|
|
const describeFn = vi.fn().mockResolvedValue(empty);
|
|
const recycleAx = vi.fn().mockResolvedValue(undefined);
|
|
const res = await run(makeApi(describeFn, recycleAx));
|
|
|
|
// 3 transition-window probes + 1 post-recycle probe.
|
|
expect(describeFn).toHaveBeenCalledTimes(4);
|
|
expect(recycleAx).toHaveBeenCalledTimes(1);
|
|
expect(res.description).toContain("Focusable: (none reported)");
|
|
expect(res.hint).toMatch(/still launching|loading|transition|recycl/i);
|
|
expect(res.description).toContain("Note:");
|
|
});
|
|
});
|
|
|
|
// Android TV target — platform "android" by serial shape.
|
|
const ANDROID_TV_DEVICE: DeviceInfo = {
|
|
id: "emulator-5556",
|
|
platform: "android",
|
|
kind: "emulator",
|
|
};
|
|
|
|
describe("describe (TV) — Android skips the focus-engine retries", () => {
|
|
it("does not retry or recycle on an empty Android focus set — one probe then the uiautomator fallback", async () => {
|
|
// On Android TV an empty focus set is steady state, not a transition — the
|
|
// retry loop and the no-op recycle would just repeat the empty dump. One
|
|
// probe, then straight to the full-tree fallback.
|
|
const describeFn = vi.fn().mockResolvedValue(empty);
|
|
const recycleAx = vi.fn().mockResolvedValue(undefined);
|
|
const frame = { x: 0, y: 0, width: 100, height: 50 };
|
|
describeAndroidMock.mockResolvedValue({
|
|
tree: {
|
|
role: "RCTView",
|
|
frame,
|
|
children: [{ role: "AXButton", label: "Play", frame, children: [] }],
|
|
},
|
|
source: "uiautomator",
|
|
});
|
|
|
|
const res = await describeTv(makeRegistry(makeApi(describeFn, recycleAx)), ANDROID_TV_DEVICE);
|
|
|
|
// Exactly one probe before the fallback — no retry loop, no recycle.
|
|
expect(recycleAx).not.toHaveBeenCalled();
|
|
expect(describeFn).toHaveBeenCalledTimes(1);
|
|
// The Android uiautomator fallback supplied the rendering + its hint.
|
|
expect(describeAndroidMock).toHaveBeenCalledTimes(1);
|
|
expect(res.hint).toMatch(/Android TV focus engine/i);
|
|
});
|
|
});
|