Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ export type BackendSnapshotResult = {

export type BackendSnapshotOptions = SnapshotOptions & {
includeRects?: boolean;
includeHiddenContentHints?: boolean;
outPath?: string;
};

Expand Down
4 changes: 4 additions & 0 deletions src/commands/interaction/runtime/selector-read-shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ export async function captureSelectorSnapshot(
scope?: string;
includeRects?: boolean;
interactiveOnly?: boolean;
includeHiddenContentHints?: boolean;
} = {
updateSession: true,
},
Expand All @@ -67,6 +68,9 @@ export async function captureSelectorSnapshot(
scope: captureOptions.scope ?? options.scope,
raw: options.raw,
includeRects: captureOptions.includeRects,
...(captureOptions.includeHiddenContentHints !== undefined
? { includeHiddenContentHints: captureOptions.includeHiddenContentHints }
: {}),
});
const snapshot =
result.snapshot ??
Expand Down
44 changes: 44 additions & 0 deletions src/commands/interaction/runtime/selector-read.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,50 @@ test('runtime find wait reports sparse snapshot verdicts on the selector-read ro
assert.equal(session.snapshot, initialSnapshot);
});

test('runtime find wait skips hidden-content hint derivation on every poll (#1270)', async () => {
// A `wait`'s presence check never consumes scroll hints, so every poll must ask the capture
// layer to skip deriving them — otherwise a single pathological `dumpsys activity top` call
// can eat the whole wait budget from inside this loop.
const snapshot = selectorReadSnapshot();
const captureOptionsCalls: Array<BackendSnapshotOptions | undefined> = [];
let elapsed = 0;
const device = createAgentDevice({
backend: {
platform: 'android',
captureSnapshot: async (_context, options) => {
captureOptionsCalls.push(options);
return { snapshot };
},
} satisfies AgentDeviceBackend,
artifacts: createLocalArtifactAdapter(),
sessions: createMemorySessionStore([{ name: 'default', snapshot }]),
policy: localCommandPolicy(),
clock: {
now: () => elapsed,
sleep: async () => {
elapsed += 300;
},
},
});

await assert.rejects(
() =>
device.selectors.find({
session: 'default',
locator: 'text',
query: 'Never appears',
action: 'wait',
timeoutMs: 500,
}),
/find wait timed out/,
);

assert.equal(captureOptionsCalls.length >= 2, true);
for (const options of captureOptionsCalls) {
assert.equal(options?.includeHiddenContentHints, false);
}
});

test('runtime wait can use backend text search', async () => {
const device = createSelectorDevice(selectorReadSnapshot(), {
findText: true,
Expand Down
17 changes: 15 additions & 2 deletions src/commands/interaction/runtime/selector-read.ts
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,12 @@ async function waitForFindMatch(
const timeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const start = now(runtime);
while (now(runtime) - start < timeout) {
const { match } = await findFirstLocatorMatch(runtime, options, locator);
// A presence check never consumes scroll hints, so every poll skips deriving them —
// otherwise a single pathological `dumpsys activity top` call can eat the whole wait
// budget from inside this loop (#1270).
const { match } = await findFirstLocatorMatch(runtime, options, locator, {
includeHiddenContentHints: false,
});
if (match) return { kind: 'found', found: true, waitedMs: now(runtime) - start };
await sleep(runtime, POLL_INTERVAL_MS);
}
Expand All @@ -459,11 +464,13 @@ async function findFirstLocatorMatch(
runtime: AgentDeviceRuntime,
options: FindReadCommandOptions,
locator: FindLocator,
captureOverrides?: { includeHiddenContentHints?: boolean },
): Promise<{ capture: CapturedSnapshot; match: SnapshotNode | undefined }> {
const selectorChain = parseFindSelectorExpression(locator, options.query);
const capture = await captureSelectorSnapshot(runtime, options, {
updateSession: true,
scope: findSnapshotScope(runtime, locator, options.query, selectorChain),
includeHiddenContentHints: captureOverrides?.includeHiddenContentHints,
...deriveSelectorCapturePolicy({ selectorChain }),
});
if (isSparseSnapshotQualityVerdict(capture.snapshot.snapshotQuality)) {
Expand Down Expand Up @@ -512,8 +519,10 @@ async function waitForSelector(
const chain = parseSelectorChain(selectorExpression);
const capturePolicy = deriveSelectorCapturePolicy({ selectorChain: chain });
while (now(runtime) - start < timeout) {
// Presence-only poll: skip scroll-hint derivation (#1270), same as waitForFindMatch.
const capture = await captureSelectorSnapshot(runtime, options, {
updateSession: true,
includeHiddenContentHints: false,
...capturePolicy,
});
const match = findSelectorChainMatch(capture.snapshot.nodes, chain, {
Expand Down Expand Up @@ -549,7 +558,11 @@ async function snapshotContainsText(
options: WaitCommandOptions,
text: string,
): Promise<boolean> {
const capture = await captureSelectorSnapshot(runtime, options, { updateSession: true });
// Presence-only poll: skip scroll-hint derivation (#1270), same as waitForFindMatch.
const capture = await captureSelectorSnapshot(runtime, options, {
updateSession: true,
includeHiddenContentHints: false,
});
return Boolean(findNodeByLabel(capture.snapshot.nodes, text));
}

Expand Down
2 changes: 2 additions & 0 deletions src/core/dispatch-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ export type CommandFlags = Omit<CliFlags, DaemonExcludedCliFlag> & {
kind?: string;
maestro?: MaestroRuntimeFlags;
postGestureStabilization?: boolean;
snapshotIncludeHiddenContentHints?: boolean;
leaseProvider?: string;
provider?: string;
deviceKey?: string;
Expand Down Expand Up @@ -61,6 +62,7 @@ export type DispatchContext = ScreenshotDispatchFlags & {
snapshotScope?: string;
snapshotRaw?: boolean;
snapshotIncludeRects?: boolean;
snapshotIncludeHiddenContentHints?: boolean;
skipIosSimulatorBootCheck?: boolean;
count?: number;
intervalMs?: number;
Expand Down
1 change: 1 addition & 0 deletions src/core/dispatch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -636,6 +636,7 @@ async function handleSnapshotCommand(
scope: context?.snapshotScope,
raw: context?.snapshotRaw,
includeRects: context?.snapshotIncludeRects,
includeHiddenContentHints: context?.snapshotIncludeHiddenContentHints,
surface: context?.surface,
});
}
Expand Down
1 change: 1 addition & 0 deletions src/core/interactor-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ export const MAESTRO_NON_HITTABLE_FALLBACK_MESSAGE = 'tapped via non-hittable co
export type SnapshotOptions = BaseSnapshotOptions & {
appBundleId?: string;
includeRects?: boolean;
includeHiddenContentHints?: boolean;
surface?: SessionSurface;
};

Expand Down
1 change: 1 addition & 0 deletions src/core/interactors/android.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ export function createAndroidInteractor(device: DeviceInfo): Interactor {
depth: options?.depth,
scope: options?.scope,
raw: options?.raw,
includeHiddenContentHints: options?.includeHiddenContentHints,
// appBundleId is present for app-backed daemon sessions; keep the helper warm there,
// but release it after standalone device snapshots so UiAutomation is not squatted.
helperSessionScope: options?.appBundleId ? 'daemon-session' : 'command',
Expand Down
1 change: 1 addition & 0 deletions src/daemon/context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ export function contextFromFlags(
snapshotDepth: flags?.snapshotDepth,
snapshotScope: flags?.snapshotScope,
snapshotRaw: flags?.snapshotRaw,
snapshotIncludeHiddenContentHints: flags?.snapshotIncludeHiddenContentHints,
...screenshotFlagsFromOptions(flags),
count: flags?.count,
intervalMs: flags?.intervalMs,
Expand Down
73 changes: 73 additions & 0 deletions src/daemon/handlers/__tests__/snapshot-handler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1543,6 +1543,79 @@ test('wait selector timeout includes compact current-surface details', async ()
}
});

test('wait selector polling skips hidden-content hint derivation on every poll (#1270)', async () => {
// The #1270 repro: `wait 'label="Battery"' 8000` on Android. A presence-only wait never
// consumes scroll hints, so every per-poll snapshot capture must disable hint derivation —
// otherwise a pathological `dumpsys activity top` call is charged against the wait budget.
const sessionName = 'android-wait-selector-skips-hints';
const withoutBattery = {
nodes: locationRequiredNodes,
truncated: false,
backend: 'android',
analysis: { rawNodeCount: 2, maxDepth: 0 },
};
const withBattery = {
nodes: [
{
index: 0,
depth: 0,
type: 'android.widget.TextView',
label: 'Battery',
rect: { x: 252, y: 780, width: 153, height: 65 },
},
],
truncated: false,
backend: 'android',
analysis: { rawNodeCount: 1, maxDepth: 0 },
};
mockDispatch.mockResolvedValueOnce(withoutBattery).mockResolvedValueOnce(withBattery);

const response = await runWaitCommand(sessionName, androidDevice, ['label="Battery"', '8000']);

expect(response?.ok).toBe(true);
const snapshotCalls = mockDispatch.mock.calls.filter(([, command]) => command === 'snapshot');
expect(snapshotCalls.length).toBe(2);
for (const call of snapshotCalls) {
const context = call[4] as { snapshotIncludeHiddenContentHints?: boolean } | undefined;
expect(context?.snapshotIncludeHiddenContentHints).toBe(false);
}
});

test('wait text polling skips hidden-content hint derivation on every poll (#1270)', async () => {
const sessionName = 'android-wait-text-skips-hints';
const withoutBattery = {
nodes: locationRequiredNodes,
truncated: false,
backend: 'android',
analysis: { rawNodeCount: 2, maxDepth: 0 },
};
const withBattery = {
nodes: [
{
index: 0,
depth: 0,
type: 'android.widget.TextView',
label: 'Battery',
rect: { x: 252, y: 780, width: 153, height: 65 },
},
],
truncated: false,
backend: 'android',
analysis: { rawNodeCount: 1, maxDepth: 0 },
};
mockDispatch.mockResolvedValueOnce(withoutBattery).mockResolvedValueOnce(withBattery);

const response = await runWaitCommand(sessionName, androidDevice, ['Battery', '8000']);

expect(response?.ok).toBe(true);
const snapshotCalls = mockDispatch.mock.calls.filter(([, command]) => command === 'snapshot');
expect(snapshotCalls.length).toBe(2);
for (const call of snapshotCalls) {
const context = call[4] as { snapshotIncludeHiddenContentHints?: boolean } | undefined;
expect(context?.snapshotIncludeHiddenContentHints).toBe(false);
}
});

test('wait timeout summary prefers content labels over chrome and identifier noise', async () => {
const sessionName = 'ios-wait-timeout-surface-summary';
mockRunnerCommand.mockResolvedValue({ found: false });
Expand Down
25 changes: 19 additions & 6 deletions src/daemon/selector-runtime-backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,14 @@ type AppleRunnerFindTextTarget = {
};

type SnapshotFlagOverrides = Partial<
Pick<CommandFlags, 'snapshotInteractiveOnly' | 'snapshotScope' | 'snapshotDepth' | 'snapshotRaw'>
Pick<
CommandFlags,
| 'snapshotInteractiveOnly'
| 'snapshotScope'
| 'snapshotDepth'
| 'snapshotRaw'
| 'snapshotIncludeHiddenContentHints'
>
>;

export function createSelectorRuntimeForDevice(params: SelectorRuntimeDeviceParams) {
Expand Down Expand Up @@ -145,12 +152,15 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice
}

function snapshotFlagOverrides(options: BackendSnapshotOptions | undefined): SnapshotFlagOverrides {
const { interactiveOnly, scope, depth, raw, includeHiddenContentHints } = options ?? {};
const flags: SnapshotFlagOverrides = {};
if (options?.interactiveOnly !== undefined)
flags.snapshotInteractiveOnly = options.interactiveOnly;
if (options?.scope !== undefined) flags.snapshotScope = options.scope;
if (options?.depth !== undefined) flags.snapshotDepth = options.depth;
if (options?.raw !== undefined) flags.snapshotRaw = options.raw;
if (interactiveOnly !== undefined) flags.snapshotInteractiveOnly = interactiveOnly;
if (scope !== undefined) flags.snapshotScope = scope;
if (depth !== undefined) flags.snapshotDepth = depth;
if (raw !== undefined) flags.snapshotRaw = raw;
if (includeHiddenContentHints !== undefined) {
flags.snapshotIncludeHiddenContentHints = includeHiddenContentHints;
}
return flags;
}

Expand Down Expand Up @@ -229,6 +239,9 @@ async function captureWaitSnapshot(params: SelectorRuntimeDeviceParams) {
flags: {
...params.req.flags,
snapshotInteractiveOnly: false,
// Presence-only wait poll (findText is only called from waitForText):
// skip scroll-hint derivation (#1270).
snapshotIncludeHiddenContentHints: false,
},
cache: {
forceFresh: true,
Expand Down
46 changes: 46 additions & 0 deletions src/platforms/android/__tests__/snapshot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1133,6 +1133,52 @@ test('snapshotAndroid skips activity dump when snapshot has no scrollable nodes'
assert.equal(result.nodes[0]?.label, 'Continue');
});

test('snapshotAndroid caps the activity-top scroll-hint probe at 1.5s', async () => {
const xml = `<?xml version="1.0" encoding="UTF-8"?>
<hierarchy rotation="0">
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" clickable="false" focusable="false">
<node class="android.widget.ScrollView" bounds="[0,100][390,600]" clickable="false" focusable="false">
<node class="android.widget.Button" text="Continue" bounds="[20,120][200,180]" clickable="true" focusable="true" />
</node>
</node>
</hierarchy>`;
const activityTimeouts: Array<number | undefined> = [];
const helperAdb = createHelperAdb({
instrument: async () => ({ exitCode: 0, stdout: helperOutput(xml), stderr: '' }),
activity: async (_args, options) => {
activityTimeouts.push(options?.timeoutMs);
return { exitCode: 0, stdout: '', stderr: '' };
},
});

await snapshotAndroidWithHelper(helperAdb);

assert.deepEqual(activityTimeouts, [1500]);
});

test('snapshotAndroid skips the activity-top probe when the helper XML already carries scroll-action attributes', async () => {
const xml = `<?xml version="1.0" encoding="UTF-8"?>
<hierarchy rotation="0">
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" clickable="false" focusable="false">
<node class="android.widget.ScrollView" scrollable="true" can-scroll-forward="true" can-scroll-backward="false" bounds="[0,100][390,600]" clickable="false" focusable="false">
<node class="android.widget.Button" text="Continue" bounds="[20,120][200,180]" clickable="true" focusable="true" />
</node>
</node>
</hierarchy>`;
let activityDumpCalled = false;
const helperAdb = createHelperAdb({
instrument: async () => ({ exitCode: 0, stdout: helperOutput(xml), stderr: '' }),
activity: async () => {
activityDumpCalled = true;
return { exitCode: 0, stdout: '', stderr: '' };
},
});

await snapshotAndroidWithHelper(helperAdb);

assert.equal(activityDumpCalled, false);
});

test('snapshotAndroid skips hidden content hints when disabled', async () => {
const xml = `<?xml version="1.0" encoding="UTF-8"?>
<hierarchy rotation="0">
Expand Down
7 changes: 6 additions & 1 deletion src/platforms/android/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -592,14 +592,19 @@ function collectExistingHiddenContentHints(
return hintsByIndex;
}

// Cosmetic scroll-hint metadata: healthy latency is ~37ms, but `dumpsys activity top` has been
// measured up to ~5.9s under contention. Cap it well under a typical caller timeout so one
// pathological call can't eat a whole `wait`/`get` budget (see #1270).
const ACTIVITY_TOP_TIMEOUT_MS = 1_500;

async function dumpActivityTop(
device: DeviceInfo,
adb = resolveAndroidAdbExecutor(device),
): Promise<string | null> {
try {
const result = await adb(['shell', 'dumpsys', 'activity', 'top'], {
allowFailure: true,
timeoutMs: 8_000,
timeoutMs: ACTIVITY_TOP_TIMEOUT_MS,
});
const text = `${result.stdout}\n${result.stderr}`.trim();
return text.length > 0 ? text : null;
Expand Down
Loading