skills/software-mansion__argent
1
0
Fork 0
mirror of https://github.com/software-mansion/argent.git synced 2026-09-14 19:27:14 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
fix/cpu-index-parallel-array-validation
software-mansion__argent/packages/tool-server/test/describe-android-label-preserve.test.ts
T
Ignacy Łątka d17d6883cb fix(describe): preserve accessibility label/id when collapsing Android wrappers (#440) 
## Summary

The Android uiautomator describe parser collapses redundant wrapper
nodes in two places. Both sites kept the inner/child node and silently
discarded the wrapper's own accessibility `content-desc` label and
`resource-id`, even when the surviving node had neither. The result is
that a labelled tap target becomes anonymous in the describe tree, so an
agent can no longer identify or target it.

This change makes both collapse sites merge rather than drop: the
surviving node still prefers its own label/identifier (the existing
"inner is more specific" intent), but now falls back to the wrapper's
values instead of throwing them away.

## Defect A — duplicate-wrapper collapse (`computeNodeOutput`)

When a clickable parent and its single clickable child have identical
bounds, the duplicate-wrapper collapse keeps only the inner child and
discards the parent's `content-desc`/`resource-id` — even when the inner
child has neither. A React Native `Pressable` with an
`accessibilityLabel` wrapping a bare clickable native view (label only
on the outer node) collapses to an anonymous node.

Repro:
```
<node clickable content-desc="Like, 5 likes">
  <node clickable content-desc="" />
</node>
```
- Observed: collapses to a single node with NO label.
- Expected: the collapsed node keeps `label="Like, 5 likes"`.

Fix: when collapsing, carry the wrapper's label/`resource-id` onto the
surviving child only when the child lacks its own. The existing test
that asserts an inner `inner` label wins over an outer `outer` label is
preserved (child value still takes precedence when present).

## Defect B — bounds-less single-child pass-through (`finalizeUiNode`)

A node with no parseable `bounds`, exactly one child, and its OWN
`content-desc` was replaced wholesale by its child (`if (children.length
=== 1) return children[0]`), discarding the wrapper's label/identifier.
This is reachable for the bounds-less Compose group containers the
parser already supports when such a container carries a `content-desc`.

Repro:
```
<node content-desc="Profile photo">
  <node bounds=... content-desc="avatar" />
</node>
```
- Observed: keeps only `avatar`; `Profile photo` is lost.
- Expected: both labels survive.

Fix: only pass a single child straight through when the wrapper has no
label/identifier of its own. When the wrapper carries its own identity,
keep it as a node whose frame is the union of its (sole) child — so
neither the wrapper's nor the child's label is dropped. The bounds-less
multi-child path (Compose containers with 2+ children) is unchanged.

## Tests

New regression test
`packages/tool-server/test/describe-android-label-preserve.test.ts`
covers both defects. It fails before the fix (`[]` for A, `['avatar']`
for B) and passes after. The full existing uiautomator/describe android
suites remain green.
2026-07-03 16:19:16 +02:00

100 lines
6.9 KiB
TypeScript
Raw Permalink Blame History

import { describe, it, expect } from "vitest";
import { parseUiAutomatorDump } from "../src/tools/describe/platforms/android/uiautomator-parser";
import type { DescribeNode } from "../src/tools/describe/contract";
function flatten(tree: DescribeNode): DescribeNode[] {
const out: DescribeNode[] = [];
const stack: DescribeNode[] = [tree];
while (stack.length) {
const n = stack.pop()!;
out.push(n);
for (let i = n.children.length - 1; i >= 0; i--) stack.push(n.children[i]!);
}
return out;
}
describe("describe android collapse preserves label/identifier", () => {
it("duplicate-wrapper collapse keeps the outer-only label", () => {
const xml = `<?xml version='1.0' encoding='UTF-8'?>\n<hierarchy>\n <node class="android.view.ViewGroup" bounds="[0,0][100,100]" clickable="true" content-desc="Like, 5 likes">\n <node class="android.widget.Button" bounds="[0,0][100,100]" clickable="true" content-desc="" text=""/>\n </node>\n</hierarchy>`;
const labels = flatten(parseUiAutomatorDump(xml, 100, 100))
.map((n) => n.label)
.filter(Boolean);
expect(labels).toContain("Like, 5 likes");
});
it("bounds-less single-child wrapper keeps its own label", () => {
const xml = `<?xml version='1.0' encoding='UTF-8'?>\n<hierarchy>\n <node class="android.view.View" content-desc="Profile photo">\n <node class="android.widget.ImageView" bounds="[0,0][100,100]" content-desc="avatar"/>\n </node>\n</hierarchy>`;
const labels = flatten(parseUiAutomatorDump(xml, 100, 100))
.map((n) => n.label)
.filter(Boolean);
expect(labels).toContain("Profile photo");
});
it("an id-bearing middle layout container does NOT block duplicate-wrapper collapse", () => {
// A clickable parent, a non-clickable middle container, and a clickable
// child at identical bounds must collapse to a SINGLE tap target — not
// three stacked nodes with "Submit" repeated.
const xml = `<?xml version='1.0' encoding='UTF-8'?>\n<hierarchy>\n <node class="android.view.ViewGroup" bounds="[0,0][100,50]" clickable="true">\n <node class="android.widget.FrameLayout" bounds="[0,0][100,50]" resource-id="com.app:id/wrap">\n <node class="android.widget.Button" bounds="[0,0][100,50]" clickable="true" text="Submit"/>\n </node>\n </node>\n</hierarchy>`;
const nodes = flatten(parseUiAutomatorDump(xml, 100, 50));
const submits = nodes.filter((n) => n.label === "Submit");
expect(submits).toHaveLength(1);
expect(nodes.map((n) => n.role)).not.toContain("FrameLayout");
});
it("flattens nested id-bearing layout containers down to their content", () => {
// Regression: nested id-only containers (android:id/content is on nearly
// every dump) must not each survive as a wrapper level.
const xml = `<?xml version='1.0' encoding='UTF-8'?>\n<hierarchy>\n <node class="android.widget.FrameLayout" bounds="[0,0][100,200]" resource-id="android:id/content">\n <node class="android.widget.LinearLayout" bounds="[0,0][100,200]" resource-id="com.app:id/root">\n <node class="android.widget.TextView" bounds="[0,0][100,20]" text="A"/>\n <node class="android.widget.TextView" bounds="[0,20][100,40]" text="B"/>\n </node>\n </node>\n</hierarchy>`;
const nodes = flatten(parseUiAutomatorDump(xml, 100, 200));
const roles = nodes.map((n) => n.role);
expect(roles).not.toContain("FrameLayout");
expect(roles).not.toContain("LinearLayout");
expect(nodes.filter((n) => n.role === "StaticText")).toHaveLength(2);
});
it("collapse copies the wrapper's resource-id onto an id-less child", () => {
// A clickable wrapper carrying a resource-id collapses into its single
// anonymous clickable child; the child must inherit the wrapper's id so the
// tap target stays addressable by id.
const xml = `<?xml version='1.0' encoding='UTF-8'?>\n<hierarchy>\n <node class="android.view.ViewGroup" bounds="[0,0][100,50]" clickable="true" resource-id="com.app:id/likeBtn">\n <node class="android.widget.Button" bounds="[0,0][100,50]" clickable="true" content-desc="" text=""/>\n </node>\n</hierarchy>`;
const ids = flatten(parseUiAutomatorDump(xml, 100, 50))
.map((n) => n.identifier)
.filter(Boolean);
expect(ids).toContain("com.app:id/likeBtn");
});
it("collapse never overwrites a real child id with the wrapper's (child wins)", () => {
// Both wrapper and child carry an id: the child's own id must survive, not
// be clobbered by the collapsing wrapper's.
const xml = `<?xml version='1.0' encoding='UTF-8'?>\n<hierarchy>\n <node class="android.view.ViewGroup" bounds="[0,0][100,50]" clickable="true" resource-id="com.app:id/outer">\n <node class="android.widget.Button" bounds="[0,0][100,50]" clickable="true" resource-id="com.app:id/inner" content-desc="" text=""/>\n </node>\n</hierarchy>`;
const ids = flatten(parseUiAutomatorDump(xml, 100, 50))
.map((n) => n.identifier)
.filter(Boolean);
expect(ids).toContain("com.app:id/inner");
expect(ids).not.toContain("com.app:id/outer");
});
it("propagates a label across MORE THAN ONE collapse level", () => {
// Three levels: labeled wrapper → anonymous clickable → anonymous clickable
// leaf, all at identical bounds. The label must survive both collapses onto
// the single surviving tap target.
const xml = `<?xml version='1.0' encoding='UTF-8'?>\n<hierarchy>\n <node class="android.view.ViewGroup" bounds="[0,0][100,50]" clickable="true" content-desc="Submit form">\n <node class="android.view.ViewGroup" bounds="[0,0][100,50]" clickable="true" content-desc="">\n <node class="android.widget.Button" bounds="[0,0][100,50]" clickable="true" content-desc="" text=""/>\n </node>\n </node>\n</hierarchy>`;
const labels = flatten(parseUiAutomatorDump(xml, 100, 50))
.map((n) => n.label)
.filter(Boolean);
expect(labels).toContain("Submit form");
});
it("duplicate-wrapper collapse must NOT leak an outer password field's value", () => {
// Regression: the label fallback added to the duplicate-wrapper collapse
// copies the outer node's label onto the surviving child. When the outer
// node is a password field, that label is the secret — it must be redacted
// to "[password]" first, never carried through raw. (Redaction happens at
// the point the label is derived, so the collapse's early return can't
// bypass it.)
const xml = `<?xml version='1.0' encoding='UTF-8'?>\n<hierarchy>\n <node class="android.widget.EditText" bounds="[0,0][100,100]" clickable="true" password="true" text="hunter2">\n <node class="android.view.View" bounds="[0,0][100,100]" clickable="true" text=""/>\n </node>\n</hierarchy>`;
const nodes = flatten(parseUiAutomatorDump(xml, 100, 100));
const strings = nodes.flatMap((n) =>
[n.label, n.value, n.identifier].filter((s): s is string => !!s)
);
expect(strings.join(" | ")).not.toContain("hunter2");
expect(nodes.map((n) => n.label).filter(Boolean)).toContain("[password]");
});
});
Reference in New Issue View Git Blame Copy Permalink