From 4bfd23a7ce92082236bba557c4f8aa76e0ff896d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 16 Jul 2026 11:48:32 +0200 Subject: [PATCH 1/2] fix(android): bound the scroll-hint dumpsys probe and skip it in wait polling (#1270) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deriveScrollableContentHintsIfNeeded's dumpsys activity top probe ran with an 8s timeout equal to a whole wait/get budget, and measured latency ranges from 37ms to ~5.9s under contention. Cap it at 1.5s so one pathological call can't starve a caller's timeout, and skip hint derivation entirely during find ... wait polling, since a presence check never consumes scroll hints. The wait polling loop actually lives in commands/interaction/runtime/selector-read.ts's waitForFindMatch (daemon/handlers/find.ts's own handleFindWait is unreachable for wait today — dispatchFindReadOnlyViaRuntime always intercepts read-only find actions first), so the skip-hints option threads through that capture path down to snapshotAndroid via the existing flags -> contextFromFlags -> dispatchCommand -> interactor.snapshot channel. --- src/backend.ts | 1 + .../runtime/selector-read-shared.ts | 4 ++ .../interaction/runtime/selector-read.test.ts | 44 ++++++++++++++++++ .../interaction/runtime/selector-read.ts | 9 +++- src/core/dispatch-context.ts | 2 + src/core/dispatch.ts | 1 + src/core/interactor-types.ts | 1 + src/core/interactors/android.ts | 1 + src/daemon/context.ts | 1 + src/daemon/selector-runtime-backend.ts | 22 ++++++--- .../android/__tests__/snapshot.test.ts | 46 +++++++++++++++++++ src/platforms/android/snapshot.ts | 7 ++- 12 files changed, 131 insertions(+), 8 deletions(-) diff --git a/src/backend.ts b/src/backend.ts index 90ab06d81..f204da3d2 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -63,6 +63,7 @@ export type BackendSnapshotResult = { export type BackendSnapshotOptions = SnapshotOptions & { includeRects?: boolean; + includeHiddenContentHints?: boolean; outPath?: string; }; diff --git a/src/commands/interaction/runtime/selector-read-shared.ts b/src/commands/interaction/runtime/selector-read-shared.ts index 89edbd6ef..478ee5f7f 100644 --- a/src/commands/interaction/runtime/selector-read-shared.ts +++ b/src/commands/interaction/runtime/selector-read-shared.ts @@ -51,6 +51,7 @@ export async function captureSelectorSnapshot( scope?: string; includeRects?: boolean; interactiveOnly?: boolean; + includeHiddenContentHints?: boolean; } = { updateSession: true, }, @@ -67,6 +68,9 @@ export async function captureSelectorSnapshot( scope: captureOptions.scope ?? options.scope, raw: options.raw, includeRects: captureOptions.includeRects, + ...(captureOptions.includeHiddenContentHints !== undefined + ? { includeHiddenContentHints: captureOptions.includeHiddenContentHints } + : {}), }); const snapshot = result.snapshot ?? diff --git a/src/commands/interaction/runtime/selector-read.test.ts b/src/commands/interaction/runtime/selector-read.test.ts index 3f8e4071b..3fb8a1853 100644 --- a/src/commands/interaction/runtime/selector-read.test.ts +++ b/src/commands/interaction/runtime/selector-read.test.ts @@ -406,6 +406,50 @@ test('runtime find wait reports sparse snapshot verdicts on the selector-read ro assert.equal(session.snapshot, initialSnapshot); }); +test('runtime find wait skips hidden-content hint derivation on every poll (#1270)', async () => { + // A `wait`'s presence check never consumes scroll hints, so every poll must ask the capture + // layer to skip deriving them — otherwise a single pathological `dumpsys activity top` call + // can eat the whole wait budget from inside this loop. + const snapshot = selectorReadSnapshot(); + const captureOptionsCalls: Array = []; + let elapsed = 0; + const device = createAgentDevice({ + backend: { + platform: 'android', + captureSnapshot: async (_context, options) => { + captureOptionsCalls.push(options); + return { snapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + clock: { + now: () => elapsed, + sleep: async () => { + elapsed += 300; + }, + }, + }); + + await assert.rejects( + () => + device.selectors.find({ + session: 'default', + locator: 'text', + query: 'Never appears', + action: 'wait', + timeoutMs: 500, + }), + /find wait timed out/, + ); + + assert.equal(captureOptionsCalls.length >= 2, true); + for (const options of captureOptionsCalls) { + assert.equal(options?.includeHiddenContentHints, false); + } +}); + test('runtime wait can use backend text search', async () => { const device = createSelectorDevice(selectorReadSnapshot(), { findText: true, diff --git a/src/commands/interaction/runtime/selector-read.ts b/src/commands/interaction/runtime/selector-read.ts index 735a3c227..5e9bf4349 100644 --- a/src/commands/interaction/runtime/selector-read.ts +++ b/src/commands/interaction/runtime/selector-read.ts @@ -448,7 +448,12 @@ async function waitForFindMatch( const timeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; const start = now(runtime); while (now(runtime) - start < timeout) { - const { match } = await findFirstLocatorMatch(runtime, options, locator); + // A presence check never consumes scroll hints, so every poll skips deriving them — + // otherwise a single pathological `dumpsys activity top` call can eat the whole wait + // budget from inside this loop (#1270). + const { match } = await findFirstLocatorMatch(runtime, options, locator, { + includeHiddenContentHints: false, + }); if (match) return { kind: 'found', found: true, waitedMs: now(runtime) - start }; await sleep(runtime, POLL_INTERVAL_MS); } @@ -459,11 +464,13 @@ async function findFirstLocatorMatch( runtime: AgentDeviceRuntime, options: FindReadCommandOptions, locator: FindLocator, + captureOverrides?: { includeHiddenContentHints?: boolean }, ): Promise<{ capture: CapturedSnapshot; match: SnapshotNode | undefined }> { const selectorChain = parseFindSelectorExpression(locator, options.query); const capture = await captureSelectorSnapshot(runtime, options, { updateSession: true, scope: findSnapshotScope(runtime, locator, options.query, selectorChain), + includeHiddenContentHints: captureOverrides?.includeHiddenContentHints, ...deriveSelectorCapturePolicy({ selectorChain }), }); if (isSparseSnapshotQualityVerdict(capture.snapshot.snapshotQuality)) { diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index 2b665bfa7..ad2915043 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -26,6 +26,7 @@ export type CommandFlags = Omit & { kind?: string; maestro?: MaestroRuntimeFlags; postGestureStabilization?: boolean; + snapshotIncludeHiddenContentHints?: boolean; leaseProvider?: string; provider?: string; deviceKey?: string; @@ -61,6 +62,7 @@ export type DispatchContext = ScreenshotDispatchFlags & { snapshotScope?: string; snapshotRaw?: boolean; snapshotIncludeRects?: boolean; + snapshotIncludeHiddenContentHints?: boolean; skipIosSimulatorBootCheck?: boolean; count?: number; intervalMs?: number; diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 02add1da8..286250f00 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -636,6 +636,7 @@ async function handleSnapshotCommand( scope: context?.snapshotScope, raw: context?.snapshotRaw, includeRects: context?.snapshotIncludeRects, + includeHiddenContentHints: context?.snapshotIncludeHiddenContentHints, surface: context?.surface, }); } diff --git a/src/core/interactor-types.ts b/src/core/interactor-types.ts index 3a849ac00..132fc347f 100644 --- a/src/core/interactor-types.ts +++ b/src/core/interactor-types.ts @@ -72,6 +72,7 @@ export const MAESTRO_NON_HITTABLE_FALLBACK_MESSAGE = 'tapped via non-hittable co export type SnapshotOptions = BaseSnapshotOptions & { appBundleId?: string; includeRects?: boolean; + includeHiddenContentHints?: boolean; surface?: SessionSurface; }; diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index 543514885..89fcf82d3 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -66,6 +66,7 @@ export function createAndroidInteractor(device: DeviceInfo): Interactor { depth: options?.depth, scope: options?.scope, raw: options?.raw, + includeHiddenContentHints: options?.includeHiddenContentHints, // appBundleId is present for app-backed daemon sessions; keep the helper warm there, // but release it after standalone device snapshots so UiAutomation is not squatted. helperSessionScope: options?.appBundleId ? 'daemon-session' : 'command', diff --git a/src/daemon/context.ts b/src/daemon/context.ts index 6df2d1d9d..e8181af6f 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -41,6 +41,7 @@ export function contextFromFlags( snapshotDepth: flags?.snapshotDepth, snapshotScope: flags?.snapshotScope, snapshotRaw: flags?.snapshotRaw, + snapshotIncludeHiddenContentHints: flags?.snapshotIncludeHiddenContentHints, ...screenshotFlagsFromOptions(flags), count: flags?.count, intervalMs: flags?.intervalMs, diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index 4a048391f..edb4c7c43 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -42,7 +42,14 @@ type AppleRunnerFindTextTarget = { }; type SnapshotFlagOverrides = Partial< - Pick + Pick< + CommandFlags, + | 'snapshotInteractiveOnly' + | 'snapshotScope' + | 'snapshotDepth' + | 'snapshotRaw' + | 'snapshotIncludeHiddenContentHints' + > >; export function createSelectorRuntimeForDevice(params: SelectorRuntimeDeviceParams) { @@ -145,12 +152,15 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice } function snapshotFlagOverrides(options: BackendSnapshotOptions | undefined): SnapshotFlagOverrides { + const { interactiveOnly, scope, depth, raw, includeHiddenContentHints } = options ?? {}; const flags: SnapshotFlagOverrides = {}; - if (options?.interactiveOnly !== undefined) - flags.snapshotInteractiveOnly = options.interactiveOnly; - if (options?.scope !== undefined) flags.snapshotScope = options.scope; - if (options?.depth !== undefined) flags.snapshotDepth = options.depth; - if (options?.raw !== undefined) flags.snapshotRaw = options.raw; + if (interactiveOnly !== undefined) flags.snapshotInteractiveOnly = interactiveOnly; + if (scope !== undefined) flags.snapshotScope = scope; + if (depth !== undefined) flags.snapshotDepth = depth; + if (raw !== undefined) flags.snapshotRaw = raw; + if (includeHiddenContentHints !== undefined) { + flags.snapshotIncludeHiddenContentHints = includeHiddenContentHints; + } return flags; } diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index 3e6752a15..8644bf7a4 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -1133,6 +1133,52 @@ test('snapshotAndroid skips activity dump when snapshot has no scrollable nodes' assert.equal(result.nodes[0]?.label, 'Continue'); }); +test('snapshotAndroid caps the activity-top scroll-hint probe at 1.5s', async () => { + const xml = ` + + + + + + +`; + const activityTimeouts: Array = []; + const helperAdb = createHelperAdb({ + instrument: async () => ({ exitCode: 0, stdout: helperOutput(xml), stderr: '' }), + activity: async (_args, options) => { + activityTimeouts.push(options?.timeoutMs); + return { exitCode: 0, stdout: '', stderr: '' }; + }, + }); + + await snapshotAndroidWithHelper(helperAdb); + + assert.deepEqual(activityTimeouts, [1500]); +}); + +test('snapshotAndroid skips the activity-top probe when the helper XML already carries scroll-action attributes', async () => { + const xml = ` + + + + + + +`; + let activityDumpCalled = false; + const helperAdb = createHelperAdb({ + instrument: async () => ({ exitCode: 0, stdout: helperOutput(xml), stderr: '' }), + activity: async () => { + activityDumpCalled = true; + return { exitCode: 0, stdout: '', stderr: '' }; + }, + }); + + await snapshotAndroidWithHelper(helperAdb); + + assert.equal(activityDumpCalled, false); +}); + test('snapshotAndroid skips hidden content hints when disabled', async () => { const xml = ` diff --git a/src/platforms/android/snapshot.ts b/src/platforms/android/snapshot.ts index 1f25979f3..d2f27081c 100644 --- a/src/platforms/android/snapshot.ts +++ b/src/platforms/android/snapshot.ts @@ -592,6 +592,11 @@ function collectExistingHiddenContentHints( return hintsByIndex; } +// Cosmetic scroll-hint metadata: healthy latency is ~37ms, but `dumpsys activity top` has been +// measured up to ~5.9s under contention. Cap it well under a typical caller timeout so one +// pathological call can't eat a whole `wait`/`get` budget (see #1270). +const ACTIVITY_TOP_TIMEOUT_MS = 1_500; + async function dumpActivityTop( device: DeviceInfo, adb = resolveAndroidAdbExecutor(device), @@ -599,7 +604,7 @@ async function dumpActivityTop( try { const result = await adb(['shell', 'dumpsys', 'activity', 'top'], { allowFailure: true, - timeoutMs: 8_000, + timeoutMs: ACTIVITY_TOP_TIMEOUT_MS, }); const text = `${result.stdout}\n${result.stderr}`.trim(); return text.length > 0 ? text : null; From 5046190f71e8202a08fbb63ba071b6c43ffb63ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 16 Jul 2026 17:27:49 +0200 Subject: [PATCH 2/2] fix(android): skip scroll-hint derivation in standalone wait polling too (#1270) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The issue's motivating repro — wait 'label="Battery"' 8000 — polls waitForSelector, not the find-wait loop, and text/ref waits poll waitForText (backend.findText -> captureWaitSnapshot on the daemon, or the snapshotContainsText fallback). All three presence-only polling captures now disable hidden-content-hint derivation, matching the Android alert-wait capture which already did. Stable wait keeps full snapshot semantics. Adds daemon-route regressions for the exact repro shape on both the selector-wait and text-wait routes, asserting every per-poll snapshot dispatch carries snapshotIncludeHiddenContentHints: false. --- .../interaction/runtime/selector-read.ts | 8 +- .../__tests__/snapshot-handler.test.ts | 73 +++++++++++++++++++ src/daemon/selector-runtime-backend.ts | 3 + 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/commands/interaction/runtime/selector-read.ts b/src/commands/interaction/runtime/selector-read.ts index 5e9bf4349..455e5bffd 100644 --- a/src/commands/interaction/runtime/selector-read.ts +++ b/src/commands/interaction/runtime/selector-read.ts @@ -519,8 +519,10 @@ async function waitForSelector( const chain = parseSelectorChain(selectorExpression); const capturePolicy = deriveSelectorCapturePolicy({ selectorChain: chain }); while (now(runtime) - start < timeout) { + // Presence-only poll: skip scroll-hint derivation (#1270), same as waitForFindMatch. const capture = await captureSelectorSnapshot(runtime, options, { updateSession: true, + includeHiddenContentHints: false, ...capturePolicy, }); const match = findSelectorChainMatch(capture.snapshot.nodes, chain, { @@ -556,7 +558,11 @@ async function snapshotContainsText( options: WaitCommandOptions, text: string, ): Promise { - const capture = await captureSelectorSnapshot(runtime, options, { updateSession: true }); + // Presence-only poll: skip scroll-hint derivation (#1270), same as waitForFindMatch. + const capture = await captureSelectorSnapshot(runtime, options, { + updateSession: true, + includeHiddenContentHints: false, + }); return Boolean(findNodeByLabel(capture.snapshot.nodes, text)); } diff --git a/src/daemon/handlers/__tests__/snapshot-handler.test.ts b/src/daemon/handlers/__tests__/snapshot-handler.test.ts index 14929fc74..f028cc1d9 100644 --- a/src/daemon/handlers/__tests__/snapshot-handler.test.ts +++ b/src/daemon/handlers/__tests__/snapshot-handler.test.ts @@ -1543,6 +1543,79 @@ test('wait selector timeout includes compact current-surface details', async () } }); +test('wait selector polling skips hidden-content hint derivation on every poll (#1270)', async () => { + // The #1270 repro: `wait 'label="Battery"' 8000` on Android. A presence-only wait never + // consumes scroll hints, so every per-poll snapshot capture must disable hint derivation — + // otherwise a pathological `dumpsys activity top` call is charged against the wait budget. + const sessionName = 'android-wait-selector-skips-hints'; + const withoutBattery = { + nodes: locationRequiredNodes, + truncated: false, + backend: 'android', + analysis: { rawNodeCount: 2, maxDepth: 0 }, + }; + const withBattery = { + nodes: [ + { + index: 0, + depth: 0, + type: 'android.widget.TextView', + label: 'Battery', + rect: { x: 252, y: 780, width: 153, height: 65 }, + }, + ], + truncated: false, + backend: 'android', + analysis: { rawNodeCount: 1, maxDepth: 0 }, + }; + mockDispatch.mockResolvedValueOnce(withoutBattery).mockResolvedValueOnce(withBattery); + + const response = await runWaitCommand(sessionName, androidDevice, ['label="Battery"', '8000']); + + expect(response?.ok).toBe(true); + const snapshotCalls = mockDispatch.mock.calls.filter(([, command]) => command === 'snapshot'); + expect(snapshotCalls.length).toBe(2); + for (const call of snapshotCalls) { + const context = call[4] as { snapshotIncludeHiddenContentHints?: boolean } | undefined; + expect(context?.snapshotIncludeHiddenContentHints).toBe(false); + } +}); + +test('wait text polling skips hidden-content hint derivation on every poll (#1270)', async () => { + const sessionName = 'android-wait-text-skips-hints'; + const withoutBattery = { + nodes: locationRequiredNodes, + truncated: false, + backend: 'android', + analysis: { rawNodeCount: 2, maxDepth: 0 }, + }; + const withBattery = { + nodes: [ + { + index: 0, + depth: 0, + type: 'android.widget.TextView', + label: 'Battery', + rect: { x: 252, y: 780, width: 153, height: 65 }, + }, + ], + truncated: false, + backend: 'android', + analysis: { rawNodeCount: 1, maxDepth: 0 }, + }; + mockDispatch.mockResolvedValueOnce(withoutBattery).mockResolvedValueOnce(withBattery); + + const response = await runWaitCommand(sessionName, androidDevice, ['Battery', '8000']); + + expect(response?.ok).toBe(true); + const snapshotCalls = mockDispatch.mock.calls.filter(([, command]) => command === 'snapshot'); + expect(snapshotCalls.length).toBe(2); + for (const call of snapshotCalls) { + const context = call[4] as { snapshotIncludeHiddenContentHints?: boolean } | undefined; + expect(context?.snapshotIncludeHiddenContentHints).toBe(false); + } +}); + test('wait timeout summary prefers content labels over chrome and identifier noise', async () => { const sessionName = 'ios-wait-timeout-surface-summary'; mockRunnerCommand.mockResolvedValue({ found: false }); diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index edb4c7c43..3d66b4166 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -239,6 +239,9 @@ async function captureWaitSnapshot(params: SelectorRuntimeDeviceParams) { flags: { ...params.req.flags, snapshotInteractiveOnly: false, + // Presence-only wait poll (findText is only called from waitForText): + // skip scroll-hint derivation (#1270). + snapshotIncludeHiddenContentHints: false, }, cache: { forceFresh: true,