diff --git a/src/commands/capture/runtime/snapshot.test.ts b/src/commands/capture/runtime/snapshot.test.ts index f5222f61f..4f42a1b43 100644 --- a/src/commands/capture/runtime/snapshot.test.ts +++ b/src/commands/capture/runtime/snapshot.test.ts @@ -149,6 +149,27 @@ test('runtime snapshot emits filtered Android guidance from backend analysis', a ]); }); +test('runtime snapshot renders the system-surface disclosure from Android annotations', async () => { + const device = createSnapshotOnlyDevice({ + nodes: [ + { ref: 'e1', index: 0, depth: 0, type: 'FrameLayout', label: 'Quick settings' }, + { ref: 'e2', index: 1, depth: 1, parentIndex: 0, type: 'Switch', label: 'Internet' }, + ], + truncated: false, + backend: 'android', + androidSnapshot: { + backend: 'android-helper', + systemSurfaceOnly: true, + }, + }); + + const result = await device.capture.snapshot({ session: 'default' }); + + assert.equal(result.warnings?.length, 1); + assert.match(String(result.warnings?.[0]), /system surface \(notification shade, quick settings/); + assert.match(String(result.warnings?.[0]), /press back or swipe up/); +}); + test('runtime snapshot warns when iOS interactive output is root-only', async () => { const device = createSnapshotOnlyDevice({ nodes: [{ ref: 'e1', index: 0, depth: 0, type: 'Application' }], diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index ce495084e..5dc6ee6ec 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -21,6 +21,7 @@ import type { SnapshotVisibility, } from '../../../kernel/snapshot.ts'; import { buildSnapshotVisibility } from '../../../snapshot/snapshot-visibility.ts'; +import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '../../../snapshot/system-surface-disclosure.ts'; import { formatReactNativeOverlayWarning } from '../../react-native/overlay.ts'; import { buildUnchangedSnapshotMetadata, @@ -243,6 +244,9 @@ function buildSnapshotWarnings(params: { const reactNativeOverlayWarning = formatReactNativeOverlayWarning(params.snapshot.nodes); if (reactNativeOverlayWarning) warnings.push(reactNativeOverlayWarning); + const systemSurfaceWarning = formatAndroidSystemSurfaceWarning(params.annotations); + if (systemSurfaceWarning) warnings.push(systemSurfaceWarning); + const recentDropWarning = formatRecentSnapshotDropWarning(params); if (recentDropWarning) warnings.push(recentDropWarning); @@ -250,6 +254,13 @@ function buildSnapshotWarnings(params: { return Array.from(new Set(warnings)); } +function formatAndroidSystemSurfaceWarning( + annotations: SnapshotCaptureAnnotations, +): string | undefined { + if (annotations.androidSnapshot?.systemSurfaceOnly !== true) return undefined; + return ANDROID_SYSTEM_SURFACE_DISCLOSURE; +} + function buildSparseIosInteractiveWarnings(params: { snapshot: SnapshotState; options: SnapshotCommandOptions; diff --git a/src/contracts/android-system-chrome.ts b/src/contracts/android-system-chrome.ts new file mode 100644 index 000000000..224227ddb --- /dev/null +++ b/src/contracts/android-system-chrome.ts @@ -0,0 +1,69 @@ +/** + * Android status-bar/navigation-bar chrome markers, shared between the settle-chrome + * classifier (`core/snapshot-chrome.ts`, #1198) and the helper content classifier + * (`platforms/android/snapshot-content-recovery.ts`). SystemUI hosts BOTH persistent chrome + * and actionable overlays (volume panel, media/output pickers, notification shade, quick + * settings), so chrome is never a package-level fact: only these status/nav-bar marker + * resource-ids classify as chrome; every other systemui surface is real content. + */ +export const ANDROID_SYSTEM_CHROME_PACKAGE = 'com.android.systemui'; + +const ANDROID_SYSTEM_CHROME_MARKER_PREFIXES = [ + 'com.android.systemui:id/status_bar', + 'com.android.systemui:id/navigation_bar', +] as const; + +/** + * Surviving status-bar/nav-bar LEAF ids (#1251). The non-raw Android walk + * (`walkUiHierarchyNode` in `platforms/android/ui-hierarchy.ts`) drops + * unlabeled/unidentified structural nodes via `shouldIncludeStructuralAndroidNode`, + * re-parenting their children upward — and that silently swallows every + * `status_bar*`/`navigation_bar*` WRAPPER node, i.e. the only nodes the prefix + * check above matches. A non-raw capture is left with just their labeled/ + * identified LEAVES (clock, battery, wifi/mobile icons, nav buttons), whose + * own resource-ids carry no `status_bar`/`navigation_bar` prefix, so the run + * loses its marker and `collectAndroidSystemChromeRunIndexes` stops dropping + * it (verified against a real `--raw` vs. default capture pair of the same + * screen). Recognize those leaves directly, by EXACT id — not prefix, to stay + * tight: nothing here should ever swallow an actionable systemui overlay like + * the volume dialog or a media/output picker, which live under unrelated ids. + * `--raw` keeps the wrapper markers, so the prefix check above stays + * load-bearing there. + * + * A more robust fix would thread the AOSP window-type constants + * (`TYPE_STATUS_BAR` = 2000, `TYPE_NAVIGATION_BAR` = 2019) already parsed in + * `readNodeAttributes` (ui-hierarchy.ts) through to the output `SnapshotNode` + * and key off that instead of resource-ids — deferred until the id-based + * approach here proves insufficient. + */ +const ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS: ReadonlySet = new Set( + [ + // status bar + 'clock', + 'battery', + 'statusIcons', + 'notificationIcons', + 'notification_icon_area', + 'system_icons', + 'cutout_space_view', + 'mobile_signal', + 'mobile_combo', + 'mobile_group', + 'wifi_signal', + 'wifi_combo', + 'wifi_group', + 'start_side_notif_and_chip_container', + // nav bar + 'back', + 'home', + 'recent_apps', + 'home_handle', + ].map((leaf) => `${ANDROID_SYSTEM_CHROME_PACKAGE}:id/${leaf}`), +); + +/** True when the resource-id marks Android status-bar or navigation-bar chrome. */ +export function isAndroidSystemChromeResourceId(resourceId: string | null | undefined): boolean { + const identifier = resourceId ?? ''; + if (ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS.has(identifier)) return true; + return ANDROID_SYSTEM_CHROME_MARKER_PREFIXES.some((prefix) => identifier.startsWith(prefix)); +} diff --git a/src/core/snapshot-chrome.ts b/src/core/snapshot-chrome.ts index bd36af76f..ff0909ba3 100644 --- a/src/core/snapshot-chrome.ts +++ b/src/core/snapshot-chrome.ts @@ -1,3 +1,7 @@ +import { + ANDROID_SYSTEM_CHROME_PACKAGE, + isAndroidSystemChromeResourceId, +} from '../contracts/android-system-chrome.ts'; import type { SnapshotNode } from '../kernel/snapshot.ts'; import { isAndroidInputMethodSnapshotNode } from '../snapshot/android-input-method-overlays.ts'; import { normalizeType } from '../utils/text-surface.ts'; @@ -205,65 +209,11 @@ function collectSubtreeIndexes( // kept. Marker set live-verified on the emulator: the status-bar window // carries `status_bar*` ids throughout while the VolumeDialog window carries // only `volume_dialog*` ids (`input_method_nav*` bars are IME-owned and -// handled by the IME tier). -const ANDROID_SYSTEM_CHROME_PACKAGE = 'com.android.systemui'; -const ANDROID_SYSTEM_CHROME_MARKER_PREFIXES = [ - 'com.android.systemui:id/status_bar', - 'com.android.systemui:id/navigation_bar', -]; - -/** - * Surviving status-bar/nav-bar LEAF ids (#1251). The non-raw Android walk - * (`walkUiHierarchyNode` in `platforms/android/ui-hierarchy.ts`) drops - * unlabeled/unidentified structural nodes via `shouldIncludeStructuralAndroidNode`, - * re-parenting their children upward — and that silently swallows every - * `status_bar*`/`navigation_bar*` WRAPPER node, i.e. the only nodes the prefix - * check above matches. A non-raw capture is left with just their labeled/ - * identified LEAVES (clock, battery, wifi/mobile icons, nav buttons), whose - * own resource-ids carry no `status_bar`/`navigation_bar` prefix, so the run - * loses its marker and `collectAndroidSystemChromeRunIndexes` stops dropping - * it (verified against a real `--raw` vs. default capture pair of the same - * screen). Recognize those leaves directly, by EXACT id — not prefix, to stay - * tight: nothing here should ever swallow an actionable systemui overlay like - * the volume dialog or a media/output picker, which live under unrelated ids. - * `--raw` keeps the wrapper markers, so the prefix check above stays - * load-bearing there. - * - * A more robust fix would thread the AOSP window-type constants - * (`TYPE_STATUS_BAR` = 2000, `TYPE_NAVIGATION_BAR` = 2019) already parsed in - * `readNodeAttributes` (ui-hierarchy.ts) through to the output `SnapshotNode` - * and key off that instead of resource-ids — deferred until the id-based - * approach here proves insufficient. - */ -const ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS = new Set( - [ - // status bar - 'clock', - 'battery', - 'statusIcons', - 'notificationIcons', - 'notification_icon_area', - 'system_icons', - 'cutout_space_view', - 'mobile_signal', - 'mobile_combo', - 'mobile_group', - 'wifi_signal', - 'wifi_combo', - 'wifi_group', - 'start_side_notif_and_chip_container', - // nav bar - 'back', - 'home', - 'recent_apps', - 'home_handle', - ].map((leaf) => `${ANDROID_SYSTEM_CHROME_PACKAGE}:id/${leaf}`), -); - +// handled by the IME tier). The marker constants and the resource-id-level +// predicate live in `contracts/android-system-chrome.ts` so the Android +// helper content classifier reuses the same classification. function hasAndroidSystemChromeMarker(node: SnapshotNode): boolean { - const identifier = node.identifier ?? ''; - if (ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS.has(identifier)) return true; - return ANDROID_SYSTEM_CHROME_MARKER_PREFIXES.some((prefix) => identifier.startsWith(prefix)); + return isAndroidSystemChromeResourceId(node.identifier ?? ''); } /** diff --git a/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts b/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts new file mode 100644 index 000000000..0fb992039 --- /dev/null +++ b/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts @@ -0,0 +1,242 @@ +import { test, expect, vi, beforeEach } from 'vitest'; +import { handleFindCommands } from '../find.ts'; +import { dispatchFindReadOnlyViaRuntime, dispatchWaitViaRuntime } from '../../selector-runtime.ts'; +import type { DaemonRequest, DaemonResponse } from '../../types.ts'; +import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '../../../snapshot/system-surface-disclosure.ts'; +import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; +import { makeAndroidSession } from '../../../__tests__/test-utils/session-factories.ts'; + +vi.mock('../../../core/dispatch.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + dispatchCommand: vi.fn(async () => ({})), + resolveTargetDevice: vi.fn(actual.resolveTargetDevice), + }; +}); + +vi.mock('../../device-ready.ts', () => ({ + ensureDeviceReady: vi.fn(async () => {}), +})); + +import { dispatchCommand, resolveTargetDevice } from '../../../core/dispatch.ts'; +import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/index.ts'; +import { withSystemSurfaceDisclosure } from '../system-surface-disclosure.ts'; + +const mockDispatch = vi.mocked(dispatchCommand); + +// The occluding-shade capture every scenario below consumes: no application window content, one +// active quick-settings surface. The Android capture route stamps systemSurfaceOnly on both the +// annotations and the SnapshotState (see snapshot-capture.ts), so selector routes must disclose it. +const SHADE_SNAPSHOT_DATA = { + backend: 'android', + nodes: [ + { + index: 0, + depth: 0, + type: 'FrameLayout', + label: 'Quick settings', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Switch', + label: 'Internet', + hittable: true, + rect: { x: 24, y: 120, width: 156, height: 80 }, + }, + ], + androidSnapshot: { backend: 'android-helper', systemSurfaceOnly: true }, +}; + +beforeEach(() => { + mockDispatch.mockReset(); + mockDispatch.mockImplementation(async (_device: unknown, command: string) => { + return command === 'snapshot' ? SHADE_SNAPSHOT_DATA : {}; + }); +}); + +test('mutating find on a system-surface capture discloses the occlusion on the found outcome', async () => { + const sessionStore = makeSessionStore(); + const session = makeAndroidSession('default'); + sessionStore.set('default', session); + + const response = await handleFindCommands({ + req: { + token: 't', + session: 'default', + command: 'find', + positionals: ['Internet', 'click'], + flags: {}, + }, + sessionName: 'default', + logPath: '/tmp/test.log', + sessionStore, + invoke: async () => ({ ok: true, data: {} }) as DaemonResponse, + }); + + expect(response?.ok).toBe(true); + if (!response?.ok) return; + expect(String((response.data as Record).warning)).toContain( + ANDROID_SYSTEM_SURFACE_DISCLOSURE, + ); +}); + +test('read-only find exists on a system-surface capture discloses the occlusion', async () => { + const sessionStore = makeSessionStore(); + const session = makeAndroidSession('default'); + sessionStore.set('default', session); + + const response = await dispatchFindReadOnlyViaRuntime({ + req: { + token: 't', + session: 'default', + command: 'find', + positionals: ['Internet', 'exists'], + flags: {}, + } as DaemonRequest, + sessionName: 'default', + logPath: '/tmp/test.log', + sessionStore, + }); + + expect(response?.ok).toBe(true); + if (!response?.ok) return; + const data = response.data as Record; + expect(data.found).toBe(true); + expect(String(data.warning)).toContain(ANDROID_SYSTEM_SURFACE_DISCLOSURE); +}); + +test('wait timeout for app text hidden behind a system surface discloses the occlusion', async () => { + const sessionStore = makeSessionStore(); + const session = makeAndroidSession('default'); + sessionStore.set('default', session); + + const response = await dispatchWaitViaRuntime({ + req: { + token: 't', + session: 'default', + command: 'wait', + positionals: ['Bakery list', '250'], + flags: {}, + } as DaemonRequest, + sessionName: 'default', + logPath: '/tmp/test.log', + sessionStore, + }); + + expect(response.ok).toBe(false); + if (response.ok) return; + expect(response.error.message).toMatch(/wait timed out for text: Bakery list/); + expect(String(response.error.details?.hint)).toContain(ANDROID_SYSTEM_SURFACE_DISCLOSURE); +}); + +test('sessionless read-only find still discloses the occluding system surface', async () => { + vi.mocked(resolveTargetDevice).mockResolvedValueOnce(ANDROID_EMULATOR); + const sessionStore = makeSessionStore(); + + const response = await dispatchFindReadOnlyViaRuntime({ + req: { + token: 't', + session: 'default', + command: 'find', + positionals: ['Internet', 'exists'], + flags: {}, + } as DaemonRequest, + sessionName: 'default', + logPath: '/tmp/test.log', + sessionStore, + }); + + expect(response?.ok).toBe(true); + if (!response?.ok) return; + const data = response.data as Record; + expect(data.found).toBe(true); + // No session record exists to read the capture back from: the disclosure must come from the + // consumed capture itself. + expect(sessionStore.get('default')).toBeUndefined(); + expect(String(data.warning)).toContain(ANDROID_SYSTEM_SURFACE_DISCLOSURE); +}); + +test('disclosure appends after an existing success warning instead of replacing it', () => { + const response = withSystemSurfaceDisclosure( + { ok: true, data: { found: true, warning: 'prior warning text' } }, + { systemSurfaceOnly: true }, + ); + expect(response.ok).toBe(true); + if (!response.ok) return; + const warning = String((response.data as Record).warning); + expect(warning).toContain('prior warning text'); + expect(warning).toContain(ANDROID_SYSTEM_SURFACE_DISCLOSURE); + expect(warning.indexOf('prior warning text')).toBeLessThan( + warning.indexOf(ANDROID_SYSTEM_SURFACE_DISCLOSURE), + ); +}); + +test('disclosure appends after an existing failure hint instead of replacing it', () => { + const response = withSystemSurfaceDisclosure( + { + ok: false, + error: { code: 'NOT_FOUND', message: 'no match', details: { hint: 'prior hint text' } }, + }, + { systemSurfaceOnly: true }, + ); + expect(response.ok).toBe(false); + if (response.ok) return; + const hint = String(response.error.details?.hint); + expect(hint).toContain('prior hint text'); + expect(hint).toContain(ANDROID_SYSTEM_SURFACE_DISCLOSURE); + expect(hint.indexOf('prior hint text')).toBeLessThan( + hint.indexOf(ANDROID_SYSTEM_SURFACE_DISCLOSURE), + ); +}); + +test('sessionless wait success on shade content still discloses the occluding system surface', async () => { + vi.mocked(resolveTargetDevice).mockResolvedValueOnce(ANDROID_EMULATOR); + const sessionStore = makeSessionStore(); + + const response = await dispatchWaitViaRuntime({ + req: { + token: 't', + session: 'default', + command: 'wait', + positionals: ['Internet', '250'], + flags: {}, + } as DaemonRequest, + sessionName: 'default', + logPath: '/tmp/test.log', + sessionStore, + }); + + expect(response.ok).toBe(true); + if (!response.ok) return; + expect(sessionStore.get('default')).toBeUndefined(); + expect(String((response.data as Record).warning)).toContain( + ANDROID_SYSTEM_SURFACE_DISCLOSURE, + ); +}); + +test('sessionless wait timeout still discloses the occluding system surface', async () => { + vi.mocked(resolveTargetDevice).mockResolvedValueOnce(ANDROID_EMULATOR); + const sessionStore = makeSessionStore(); + + const response = await dispatchWaitViaRuntime({ + req: { + token: 't', + session: 'default', + command: 'wait', + positionals: ['Bakery list', '250'], + flags: {}, + } as DaemonRequest, + sessionName: 'default', + logPath: '/tmp/test.log', + sessionStore, + }); + + expect(response.ok).toBe(false); + if (response.ok) return; + expect(sessionStore.get('default')).toBeUndefined(); + expect(String(response.error.details?.hint)).toContain(ANDROID_SYSTEM_SURFACE_DISCLOSURE); +}); diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index a3e7fcf23..c051b9ba3 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -18,6 +18,7 @@ import { import { isSnapshotNodeInteractionBlocked } from '../../snapshot/snapshot-occlusion.ts'; import { readCommandMessage, successText } from '../../utils/success-text.ts'; import { errorResponse, noActiveSessionError } from './response.ts'; +import { withSystemSurfaceDisclosure } from './system-surface-disclosure.ts'; import { recordSessionAction } from './handler-utils.ts'; import { stripInternalInteractionFlags } from '../interaction-outcome-policy.ts'; import { dispatchFindReadOnlyViaRuntime } from '../selector-runtime.ts'; @@ -129,14 +130,17 @@ export async function handleFindCommands(params: { flags: req.flags, platform: device.platform, }); - if (!matchResult.ok) return matchResult.response; + // Matched and unmatched outcomes both consumed this capture: when it is an occluding system + // surface, the response must disclose that app content is occluded. + if (!matchResult.ok) return withSystemSurfaceDisclosure(matchResult.response, snapshotResult); const node = matchResult.node; const resolvedNode = resolveInteractiveMatchNode(nodes, node); const ref = `@${resolvedNode.ref}`; const actionFlags = { ...(req.flags ?? {}), noRecord: true }; const match: ResolvedMatch = { node, resolvedNode, ref, nodes, actionFlags }; - return dispatchFindAction(ctx, match, action, value); + const response = await dispatchFindAction(ctx, match, action, value); + return response ? withSystemSurfaceDisclosure(response, snapshotResult) : response; } /** @@ -167,6 +171,7 @@ async function dispatchFindAction( type FindSnapshotResult = { nodes: SnapshotState['nodes']; snapshotQuality?: SnapshotQualityVerdict; + systemSurfaceOnly?: boolean; }; type FindNodeFetcher = () => Promise; @@ -213,6 +218,7 @@ function createFindNodeFetcher(params: { return { nodes: snapshot.nodes, snapshotQuality: snapshot.snapshotQuality, + systemSurfaceOnly: snapshot.systemSurfaceOnly, }; }; } diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index f852643e0..2fff15b0b 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -315,11 +315,15 @@ async function captureSnapshotAttempt(params: CaptureSnapshotParams): Promise | undefined, +): DaemonResponse { + const disclosure = systemSurfaceDisclosure(snapshot); + if (!disclosure) return response; + if (response.ok) { + const warning = appended(response.data?.warning, disclosure); + return { ...response, data: { ...response.data, warning } }; + } + const details = response.error.details ?? {}; + return { + ...response, + error: { ...response.error, details: { ...details, hint: appended(details.hint, disclosure) } }, + }; +} + +function appended(existing: unknown, disclosure: string): string { + return typeof existing === 'string' && existing.trim() !== '' + ? `${existing}\n${disclosure}` + : disclosure; +} diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index 75974d138..301d21cec 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -21,6 +21,9 @@ type SelectorCaptureRuntimeParams = { sessionName: string; req: DaemonRequest; logPath?: string; + // Sessionless routes have no session record to read the consumed capture back from, so the + // capture runtime reports every consumed snapshot here for response-level disclosures. + consumedSnapshot?: { state?: SnapshotState }; }; /** @@ -62,6 +65,11 @@ export function createSelectorCaptureRuntime(params: SelectorCaptureRuntimeParam let lastSnapshotResult: SelectorCaptureResult | undefined; let lastSnapshotCacheKey: string | undefined; + const remember = (result: SelectorCaptureResult): SelectorCaptureResult => { + if (params.consumedSnapshot) params.consumedSnapshot.state = result.snapshot; + return result; + }; + const capture = async (request: SelectorCaptureRequest): Promise => { const timestamp = Date.now(); const cacheKey = selectorCaptureCacheKey(request, params.req.flags?.out); @@ -75,7 +83,7 @@ export function createSelectorCaptureRuntime(params: SelectorCaptureRuntimeParam cacheKey, }); if (reusableLastSnapshot) { - return reusableLastSnapshot; + return remember(reusableLastSnapshot); } const sessionSnapshot = reusableSessionSnapshot({ session, timestamp, request }); @@ -83,7 +91,7 @@ export function createSelectorCaptureRuntime(params: SelectorCaptureRuntimeParam lastSnapshotAt = sessionSnapshot.createdAt; lastSnapshotResult = { snapshot: sessionSnapshot }; lastSnapshotCacheKey = cacheKey; - return lastSnapshotResult; + return remember(lastSnapshotResult); } const snapshot = await captureSelectorSnapshot({ params, request }); @@ -92,7 +100,7 @@ export function createSelectorCaptureRuntime(params: SelectorCaptureRuntimeParam lastSnapshotAt = timestamp; lastSnapshotResult = result; lastSnapshotCacheKey = cacheKey; - return result; + return remember(result); }; return { capture }; diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index 2a59a9404..4b54313e9 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -7,7 +7,7 @@ import { resolveTargetDevice, type CommandFlags } from '../core/dispatch.ts'; import { createAgentDevice } from '../runtime.ts'; import { isMacOs, isApplePlatform, publicPlatformString } from '../kernel/device.ts'; import { noActiveSessionError, requireCommandSupported } from './handlers/response.ts'; -import type { SnapshotNode } from '../kernel/snapshot.ts'; +import type { SnapshotState, SnapshotNode } from '../kernel/snapshot.ts'; import { findNodeByLabel } from '../snapshot/snapshot-processing.ts'; import { runAppleRunnerCommand } from '../platforms/apple/core/runner/runner-client.ts'; import { buildAppleRunnerRequestOptions } from './apple-runner-options.ts'; @@ -29,6 +29,9 @@ export type SelectorRuntimeParams = { logPath?: string; sessionStore: SessionStore; contextFromFlags?: ContextFromFlags; + // Filled by the capture runtime with the snapshot each selector command actually consumed; + // sessionless routes disclose from here because no session record stores the capture. + consumedSnapshot?: { state?: SnapshotState }; }; type SelectorRuntimeDeviceParams = SelectorRuntimeParams & { @@ -77,6 +80,7 @@ export async function createSelectorRuntime( | { ok: true; runtime: ReturnType } | { ok: false; response: DaemonResponse } > { + params.consumedSnapshot ??= {}; const session = params.sessionStore.get(params.sessionName); if (!session && options.requireSession) { return { @@ -106,6 +110,7 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice sessionStore, sessionName, req, + consumedSnapshot: params.consumedSnapshot, logPath, }); return { @@ -236,6 +241,7 @@ async function captureWaitSnapshot(params: SelectorRuntimeDeviceParams) { sessionName: params.sessionName, req: params.req, logPath: params.logPath, + consumedSnapshot: params.consumedSnapshot, }); const { snapshot } = await captureRuntime.capture({ flags: { diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index 86cebbe25..00bf46add 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -33,6 +33,7 @@ import { toDaemonWaitData, } from './selector-recording.ts'; import { maybeWaitTimeoutSurfaceResponse } from './wait-current-surface.ts'; +import { withSystemSurfaceDisclosure } from './handlers/system-surface-disclosure.ts'; import { isDirectIosSelectorFallbackError, readSimpleIosSelectorTarget, @@ -87,7 +88,7 @@ export async function dispatchFindReadOnlyViaRuntime( }); if (!resolvedRuntime.ok) return resolvedRuntime.response; - return await toDaemonResponse(async () => { + const response = await toDaemonResponse(async () => { const result = await resolvedRuntime.runtime.selectors.find({ session: params.sessionName, requestId: req.meta?.requestId, @@ -122,6 +123,15 @@ export async function dispatchFindReadOnlyViaRuntime( } return data; }); + // The consumed capture was just stored on the session: when it is an occluding system surface, + // both found and not-found outcomes must disclose that app content is occluded. + return withSystemSurfaceDisclosure(response, consumedSessionSnapshot(params)); +} + +function consumedSessionSnapshot(params: SelectorRuntimeParams) { + // The capture runtime reports the consumed snapshot directly; sessionless selector routes have + // no session record, so the stored-session read is only a fallback for pre-captured snapshots. + return params.consumedSnapshot?.state ?? params.sessionStore.get(params.sessionName)?.snapshot; } export async function dispatchGetViaRuntime( @@ -166,7 +176,7 @@ export async function dispatchGetViaRuntime( mintedGeneration: target.refGeneration, }) : undefined; - return await toDaemonResponse(async () => { + const response = await toDaemonResponse(async () => { const result = await resolvedRuntime.runtime.selectors.get({ session: params.sessionName, requestId: req.meta?.requestId, @@ -187,6 +197,7 @@ export async function dispatchGetViaRuntime( const data = toDaemonGetData(result); return staleRefsWarning ? { ...data, warning: staleRefsWarning } : data; }); + return withSystemSurfaceDisclosure(response, consumedSessionSnapshot(params)); } export async function dispatchIsViaRuntime( @@ -234,7 +245,10 @@ export async function dispatchIsViaRuntime( recordIfSession(params.sessionStore, params.sessionName, req, result); return stripSelectorChain(result); }); - return await maybeAndroidForegroundBlockerResponse(params, response, `is ${predicate}`); + return withSystemSurfaceDisclosure( + await maybeAndroidForegroundBlockerResponse(params, response, `is ${predicate}`), + consumedSessionSnapshot(params), + ); } export async function dispatchWaitViaRuntime( @@ -277,6 +291,9 @@ export async function dispatchWaitViaRuntime( mintedGeneration: versionedRef.generation, }); } + // Wait builds its runtime directly (no createSelectorRuntime), so the consumed-snapshot slot + // must be initialized here too or sessionless waits have nowhere to report the capture from. + params.consumedSnapshot ??= {}; const execute = async () => { const runtime = createSelectorRuntimeForDevice({ ...params, @@ -300,8 +317,14 @@ export async function dispatchWaitViaRuntime( // Keep generic wait-surface details first so Android blocker detection can own the top-level message. return await maybeAndroidForegroundBlockerResponse(params, enrichedResponse, 'wait'); }; + // A pure sleep consumes no capture, so it never earns the system-surface disclosure below. if (parsed.kind === 'sleep') return await execute(); - return await withSessionlessRunnerCleanup(session, device, execute); + // Both a satisfied wait and a timeout consumed the polled capture stored on the session: + // when it is an occluding system surface, the outcome must disclose the occlusion. + return withSystemSurfaceDisclosure( + await withSessionlessRunnerCleanup(session, device, execute), + consumedSessionSnapshot(params), + ); } function readDirectIosGetSelector( diff --git a/src/kernel/snapshot.ts b/src/kernel/snapshot.ts index b3a76440b..ccad5d38c 100644 --- a/src/kernel/snapshot.ts +++ b/src/kernel/snapshot.ts @@ -109,6 +109,12 @@ export type SnapshotState = { snapshotQuality?: SnapshotQualityVerdict; comparisonSafe?: boolean; presentationKey?: string; + /** + * Android: the capture is an occluding system surface (notification shade, quick settings) + * rather than app content. Consumers that surface this tree to the agent must disclose the + * occlusion (see snapshot/system-surface-disclosure.ts). + */ + systemSurfaceOnly?: boolean; }; export type SnapshotUnchanged = { diff --git a/src/platforms/android/__tests__/snapshot-content-recovery.test.ts b/src/platforms/android/__tests__/snapshot-content-recovery.test.ts index c24d908ef..781be22d4 100644 --- a/src/platforms/android/__tests__/snapshot-content-recovery.test.ts +++ b/src/platforms/android/__tests__/snapshot-content-recovery.test.ts @@ -1,8 +1,20 @@ import { test, expect } from 'vitest'; -import { classifyAndroidHelperContentRecovery } from '../snapshot-content-recovery.ts'; +import { + classifyAndroidHelperContent, + type AndroidHelperContentRecoveryDecision, +} from '../snapshot-content-recovery.ts'; + +// Legacy assertions below check the unusable decision (or its absence); outcome-specific tests +// call classifyAndroidHelperContent directly. +function unusableDecisionFor( + ...args: Parameters +): AndroidHelperContentRecoveryDecision | undefined { + const result = classifyAndroidHelperContent(...args); + return result.outcome === 'unusable' ? result.decision : undefined; +} test('keeps known IME blocking windows instead of falling back to covered app content', () => { - const decision = classifyAndroidHelperContentRecovery( + const decision = unusableDecisionFor( helperXml([ node({ windowType: 1, @@ -40,7 +52,7 @@ test('keeps known IME blocking windows instead of falling back to covered app co }); test('rejects helper output with only one meaningful IME node', () => { - const decision = classifyAndroidHelperContentRecovery( + const decision = unusableDecisionFor( helperXml([ node({ windowType: 1, @@ -68,7 +80,7 @@ test('rejects helper output with only one meaningful IME node', () => { }); test('rejects helper output with no foreground app or IME content', () => { - const decision = classifyAndroidHelperContentRecovery( + const decision = unusableDecisionFor( helperXml([ node({ windowType: 1, @@ -96,7 +108,7 @@ test('rejects helper output with no foreground app or IME content', () => { }); test('keeps a meaningful foreground application surface owned by another package', () => { - const decision = classifyAndroidHelperContentRecovery( + const decision = unusableDecisionFor( helperXml([ node({ windowType: 1, @@ -129,7 +141,7 @@ test('keeps a meaningful foreground application surface owned by another package }); test('rejects a status-bar-only capture without window metadata', () => { - const decision = classifyAndroidHelperContentRecovery( + const decision = unusableDecisionFor( helperXml([ node({ text: '7:52', @@ -160,7 +172,7 @@ test('rejects a status-bar-only capture without window metadata', () => { }); test('rejects framework-owned content without window metadata', () => { - const decision = classifyAndroidHelperContentRecovery( + const decision = unusableDecisionFor( helperXml([ node({ text: 'System dialog', @@ -184,7 +196,7 @@ test('rejects framework-owned content without window metadata', () => { }); test('keeps a recognized system dialog without window metadata', () => { - const decision = classifyAndroidHelperContentRecovery( + const decision = unusableDecisionFor( helperXml([ node({ text: "Demo isn't responding", @@ -225,7 +237,7 @@ test('keeps a recognized system dialog without window metadata', () => { }); test('rejects system chrome inherited under an empty application window', () => { - const decision = classifyAndroidHelperContentRecovery( + const decision = unusableDecisionFor( helperXml([ node({ windowType: 1, @@ -258,6 +270,198 @@ test('rejects system chrome inherited under an empty application window', () => expect(decision?.diagnostics.helperNonSystemMeaningfulNodeCount).toBe(0); }); +test('returns a meaningful active system surface (notification shade) faithfully', () => { + const result = classifyAndroidHelperContent( + helperXml([ + node({ + windowType: 3, + windowActive: true, + packageName: 'com.android.systemui', + className: 'android.widget.FrameLayout', + }), + node({ + text: 'Wed, Jul 16', + packageName: 'com.android.systemui', + className: 'android.widget.TextView', + }), + node({ + text: 'Internet', + resourceId: 'com.android.systemui:id/qs_tile_internet', + packageName: 'com.android.systemui', + className: 'android.widget.Switch', + }), + node({ + text: 'Manage', + resourceId: 'com.android.systemui:id/manage_settings', + packageName: 'com.android.systemui', + className: 'android.widget.Button', + }), + ]), + { + backend: 'android-helper', + nodeCount: 4, + rootPresent: true, + windowCount: 1, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'com.android.settings' }, + ); + + expect(result.outcome).toBe('system-surface-only'); +}); + +test('rejects an active navigation-bar window with three-button chrome', () => { + // Back + Home + Recents are 3 meaningful nodes, but they are status/nav chrome: an active + // nav-bar window is missing-app-content residue, never a usable shade/quick-settings surface. + const result = classifyAndroidHelperContent( + helperXml([ + node({ + windowType: 3, + windowActive: true, + packageName: 'com.android.systemui', + className: 'android.widget.FrameLayout', + }), + node({ + text: 'Back', + resourceId: 'com.android.systemui:id/back', + packageName: 'com.android.systemui', + className: 'android.widget.ImageView', + }), + node({ + text: 'Home', + resourceId: 'com.android.systemui:id/home', + packageName: 'com.android.systemui', + className: 'android.widget.ImageView', + }), + node({ + text: 'Overview', + resourceId: 'com.android.systemui:id/recent_apps', + packageName: 'com.android.systemui', + className: 'android.widget.ImageView', + }), + ]), + { + backend: 'android-helper', + nodeCount: 4, + rootPresent: true, + windowCount: 1, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'com.android.settings' }, + ); + + expect(result.outcome).toBe('unusable'); +}); + +test('rejects an active status-chrome window with clock, battery, and signal icons', () => { + const result = classifyAndroidHelperContent( + helperXml([ + node({ + windowType: 3, + windowActive: true, + packageName: 'com.android.systemui', + className: 'android.widget.FrameLayout', + }), + node({ + text: '7:52', + resourceId: 'com.android.systemui:id/clock', + packageName: 'com.android.systemui', + className: 'android.widget.TextView', + }), + node({ + text: 'Battery 100 percent', + resourceId: 'com.android.systemui:id/battery', + packageName: 'com.android.systemui', + className: 'android.widget.LinearLayout', + }), + node({ + text: 'Wifi signal full', + resourceId: 'com.android.systemui:id/wifi_signal', + packageName: 'com.android.systemui', + className: 'android.widget.ImageView', + }), + ]), + { + backend: 'android-helper', + nodeCount: 4, + rootPresent: true, + windowCount: 1, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'com.android.settings' }, + ); + + expect(result.outcome).toBe('unusable'); +}); + +test('rejects a sparse active system surface below the meaningful-content floor', () => { + const result = classifyAndroidHelperContent( + helperXml([ + node({ + windowType: 3, + windowActive: true, + packageName: 'com.android.systemui', + className: 'android.widget.FrameLayout', + }), + node({ + text: '7:52', + resourceId: 'com.android.systemui:id/clock', + packageName: 'com.android.systemui', + className: 'android.widget.TextView', + }), + ]), + { + backend: 'android-helper', + nodeCount: 2, + rootPresent: true, + windowCount: 1, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'com.android.settings' }, + ); + + expect(result.outcome).toBe('unusable'); +}); + +test('rejects a content-rich system window that is neither active nor focused', () => { + const result = classifyAndroidHelperContent( + helperXml([ + node({ + windowType: 3, + packageName: 'com.android.systemui', + className: 'android.widget.FrameLayout', + }), + node({ + text: 'Wed, Jul 16', + packageName: 'com.android.systemui', + className: 'android.widget.TextView', + }), + node({ + text: 'Internet', + resourceId: 'com.android.systemui:id/qs_tile_internet', + packageName: 'com.android.systemui', + className: 'android.widget.Switch', + }), + node({ + text: 'Manage', + resourceId: 'com.android.systemui:id/manage_settings', + packageName: 'com.android.systemui', + className: 'android.widget.Button', + }), + ]), + { + backend: 'android-helper', + nodeCount: 4, + rootPresent: true, + windowCount: 1, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'com.android.settings' }, + ); + + expect(result.outcome).toBe('unusable'); +}); + function helperXml(nodes: string[]): string { return `${nodes.join('')}`; } @@ -268,6 +472,10 @@ function node(options: { packageName: string; className: string; windowType?: number; + windowActive?: boolean; }): string { - return ``; + const windowAttrs = + (options.windowType === undefined ? '' : ` window-type="${options.windowType}"`) + + (options.windowActive === undefined ? '' : ` window-active="${options.windowActive}"`); + return ``; } diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index ee32d7e7b..a5665e215 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -806,6 +806,40 @@ test('snapshotAndroid fails closed when standalone helper sees only an app overl ); }); +test('snapshotAndroid returns an occluding system surface and stamps systemSurfaceOnly', async () => { + // Notification shade / quick settings own the whole screen: no application window, but the + // active system surface carries real content. The capture is returned faithfully and the + // public metadata carries the occlusion flag that downstream disclosure warnings key off. + const helperXml = [ + '', + '', + ' ', + ' ', + ' ', + ' ', + ' ', + '', + ].join('\n'); + const helperAdb = createHelperAdb({ + instrument: async () => ({ + exitCode: 0, + stdout: helperOutput(helperXml, { nodeCount: 4 }), + stderr: '', + }), + }); + + const result = await snapshotAndroidWithHelper(helperAdb, { + appBundleId: 'com.android.settings', + }); + + assert.equal(result.androidSnapshot.backend, 'android-helper'); + assert.equal(result.androidSnapshot.systemSurfaceOnly, true); + assert.equal( + result.nodes.some((node) => node.label === 'Internet'), + true, + ); +}); + test('snapshotAndroid keeps helper output when application and system windows are both present', async () => { const helperXml = [ '', diff --git a/src/platforms/android/snapshot-content-recovery.ts b/src/platforms/android/snapshot-content-recovery.ts index 79d52882c..c33ddbf0d 100644 --- a/src/platforms/android/snapshot-content-recovery.ts +++ b/src/platforms/android/snapshot-content-recovery.ts @@ -1,5 +1,6 @@ import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; import { isAndroidInputMethodOwnedNode } from '../../contracts/android-input-ownership.ts'; +import { isAndroidSystemChromeResourceId } from '../../contracts/android-system-chrome.ts'; import { classifyAndroidAlertIdentifier } from './alert-detection.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; @@ -7,6 +8,7 @@ const ANDROID_WINDOW_TYPE_APPLICATION = 1; const MAX_REPORTED_WINDOW_TYPES = 8; const MIN_FOREGROUND_APP_MEANINGFUL_NODES = 2; const MIN_INPUT_METHOD_MEANINGFUL_NODES = 2; +const MIN_SYSTEM_SURFACE_MEANINGFUL_NODES = 3; const ANDROID_SYSTEM_PACKAGES = new Set(['android', 'com.android.systemui']); const INSUFFICIENT_APP_CONTENT_REASON = 'Android snapshot helper returned insufficient application window content'; @@ -31,16 +33,21 @@ export type AndroidHelperContentRecoveryDecision = { }; }; -export function classifyAndroidHelperContentRecovery( +export type AndroidHelperContentClassification = + | { outcome: 'ok' } + | { outcome: 'system-surface-only' } + | { outcome: 'unusable'; decision: AndroidHelperContentRecoveryDecision }; + +export function classifyAndroidHelperContent( xml: string, metadata: AndroidSnapshotBackendMetadata, options: { foregroundAppPackage?: string } = {}, -): AndroidHelperContentRecoveryDecision | undefined { - if (metadata.backend !== 'android-helper') return undefined; +): AndroidHelperContentClassification { + if (metadata.backend !== 'android-helper') return { outcome: 'ok' }; const summary = summarizeAndroidHelperXml(xml, options.foregroundAppPackage); if (isEmptyHelperOutput(summary, metadata)) { - return buildRecoveryDecision( + return unusable( summary, metadata, 'empty-helper-output', @@ -48,10 +55,14 @@ export function classifyAndroidHelperContentRecovery( ); } - if (hasRecognizedAndroidAlertSurface(summary)) return undefined; - if (isForegroundAppContentHiddenByInputMethod(summary)) return undefined; + if (hasRecognizedAndroidAlertSurface(summary)) return { outcome: 'ok' }; + // Notification shade, quick settings, and similar overlays legitimately own the whole screen: + // no application window is interactive, but the system surface carries real content. That + // capture is the truth, so it must be returned rather than classified as a helper failure. + if (isScreenOwnedByMeaningfulSystemSurface(summary)) return { outcome: 'system-surface-only' }; + if (isForegroundAppContentHiddenByInputMethod(summary)) return { outcome: 'ok' }; if (isForegroundAppContentPoor(summary)) { - return buildRecoveryDecision( + return unusable( summary, metadata, 'content-poor-app-window', @@ -60,25 +71,15 @@ export function classifyAndroidHelperContentRecovery( } if (isApplicationWindowContentPoor(summary)) { - return buildRecoveryDecision( - summary, - metadata, - 'content-poor-app-window', - INSUFFICIENT_APP_CONTENT_REASON, - ); + return unusable(summary, metadata, 'content-poor-app-window', INSUFFICIENT_APP_CONTENT_REASON); } if (isWindowlessMultiWindowContentPoor(summary, metadata)) { - return buildRecoveryDecision( - summary, - metadata, - 'content-poor-app-window', - INSUFFICIENT_APP_CONTENT_REASON, - ); + return unusable(summary, metadata, 'content-poor-app-window', INSUFFICIENT_APP_CONTENT_REASON); } if (isSystemWindowOnly(summary)) { - return buildRecoveryDecision( + return unusable( summary, metadata, 'system-window-only', @@ -86,7 +87,27 @@ export function classifyAndroidHelperContentRecovery( ); } - return undefined; + return { outcome: 'ok' }; +} + +function unusable( + summary: AndroidHelperXmlSummary, + metadata: AndroidSnapshotBackendMetadata, + reason: AndroidHelperContentRecoveryDecision['reason'], + failureReason: string, +): AndroidHelperContentClassification { + return { + outcome: 'unusable', + decision: buildRecoveryDecision(summary, metadata, reason, failureReason), + }; +} + +function isScreenOwnedByMeaningfulSystemSurface(summary: AndroidHelperXmlSummary): boolean { + return ( + summary.windowRootCount > 0 && + summary.applicationWindowRootCount === 0 && + summary.activeSystemSurfaceMeaningfulNodeCount >= MIN_SYSTEM_SURFACE_MEANINGFUL_NODES + ); } function isEmptyHelperOutput( @@ -166,6 +187,7 @@ type AndroidHelperXmlSummary = { systemUiNodeCount: number; windowRootCount: number; applicationWindowRootCount: number; + activeSystemSurfaceMeaningfulNodeCount: number; meaningfulNodeCount: number; applicationMeaningfulNodeCount: number; nonSystemMeaningfulNodeCount: number; @@ -179,6 +201,7 @@ type AndroidHelperXmlSummary = { type AndroidHelperXmlSummaryState = Omit & { currentWindowType?: number; + currentWindowActiveOrFocused?: boolean; windowTypes: Set; }; @@ -203,6 +226,7 @@ function createAndroidHelperXmlSummaryState( systemUiNodeCount: 0, windowRootCount: 0, applicationWindowRootCount: 0, + activeSystemSurfaceMeaningfulNodeCount: 0, meaningfulNodeCount: 0, applicationMeaningfulNodeCount: 0, nonSystemMeaningfulNodeCount: 0, @@ -243,6 +267,7 @@ function recordAndroidHelperWindowNode( if (node.windowType === undefined) return; summary.currentWindowType = node.windowType; + summary.currentWindowActiveOrFocused = node.windowActive === true || node.windowFocused === true; summary.windowRootCount += 1; summary.windowTypes.add(node.windowType); if (node.windowType === ANDROID_WINDOW_TYPE_APPLICATION) { @@ -257,12 +282,40 @@ function recordAndroidHelperMeaningfulNode( if (!isMeaningfulContentNode(node)) return; summary.meaningfulNodeCount += 1; + recordMeaningfulWindowOwnership(summary, node); + recordMeaningfulPackageOwnership(summary, node); +} + +function recordMeaningfulWindowOwnership( + summary: AndroidHelperXmlSummaryState, + node: AndroidUiNodeMetadata, +): void { if ( summary.currentWindowType === ANDROID_WINDOW_TYPE_APPLICATION && !isAndroidSystemPackage(node.packageName) ) { summary.applicationMeaningfulNodeCount += 1; } + // Status/nav-bar chrome must not satisfy the system-surface floor: an active navigation bar + // (Back + Home + Recents) or status chrome (clock, battery, signal icons) is missing-app-content + // residue, not a usable shade/quick-settings surface. Only non-chrome nodes count. + if (isInActiveSystemSurfaceWindow(summary) && !isAndroidSystemChromeResourceId(node.resourceId)) { + summary.activeSystemSurfaceMeaningfulNodeCount += 1; + } +} + +function isInActiveSystemSurfaceWindow(summary: AndroidHelperXmlSummaryState): boolean { + return ( + summary.currentWindowType !== undefined && + summary.currentWindowType !== ANDROID_WINDOW_TYPE_APPLICATION && + summary.currentWindowActiveOrFocused === true + ); +} + +function recordMeaningfulPackageOwnership( + summary: AndroidHelperXmlSummaryState, + node: AndroidUiNodeMetadata, +): void { if (!isAndroidSystemPackage(node.packageName)) { summary.nonSystemMeaningfulNodeCount += 1; } @@ -290,6 +343,7 @@ function finalizeAndroidHelperXmlSummary( systemUiNodeCount: summary.systemUiNodeCount, windowRootCount: summary.windowRootCount, applicationWindowRootCount: summary.applicationWindowRootCount, + activeSystemSurfaceMeaningfulNodeCount: summary.activeSystemSurfaceMeaningfulNodeCount, meaningfulNodeCount: summary.meaningfulNodeCount, applicationMeaningfulNodeCount: summary.applicationMeaningfulNodeCount, nonSystemMeaningfulNodeCount: summary.nonSystemMeaningfulNodeCount, diff --git a/src/platforms/android/snapshot-types.ts b/src/platforms/android/snapshot-types.ts index 85e537b17..a36bf3317 100644 --- a/src/platforms/android/snapshot-types.ts +++ b/src/platforms/android/snapshot-types.ts @@ -18,6 +18,7 @@ export type AndroidSnapshotBackendMetadata = { maxNodes?: number; rootPresent?: boolean; captureMode?: AndroidSnapshotCaptureMode; + systemSurfaceOnly?: boolean; windowCount?: number; nodeCount?: number; helperTruncated?: boolean; diff --git a/src/platforms/android/snapshot.ts b/src/platforms/android/snapshot.ts index 8759a20ac..cb323e1b5 100644 --- a/src/platforms/android/snapshot.ts +++ b/src/platforms/android/snapshot.ts @@ -49,7 +49,7 @@ import { } from './snapshot-helper.ts'; import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; import { - classifyAndroidHelperContentRecovery, + classifyAndroidHelperContent, type AndroidHelperContentRecoveryDecision, } from './snapshot-content-recovery.ts'; @@ -244,14 +244,22 @@ async function captureAndroidUiHierarchyWithHelper( }); } - const contentRecovery = classifyAndroidHelperContentRecovery( - helperCapture.xml, - helperCapture.metadata, - { foregroundAppPackage: options.appBundleId }, - ); - if (!contentRecovery) return helperCapture; + const content = classifyAndroidHelperContent(helperCapture.xml, helperCapture.metadata, { + foregroundAppPackage: options.appBundleId, + }); + if (content.outcome === 'system-surface-only') { + emitDiagnostic({ + phase: 'android_snapshot_helper_system_surface', + data: { foregroundAppPackage: options.appBundleId }, + }); + return { + xml: helperCapture.xml, + metadata: { ...helperCapture.metadata, systemSurfaceOnly: true }, + }; + } + if (content.outcome === 'ok') return helperCapture; return await rejectAndroidHelperContentUnavailable({ - contentRecovery, + contentRecovery: content.decision, helperDeviceKey, artifact, adb, diff --git a/src/snapshot/system-surface-disclosure.ts b/src/snapshot/system-surface-disclosure.ts new file mode 100644 index 000000000..596eea663 --- /dev/null +++ b/src/snapshot/system-surface-disclosure.ts @@ -0,0 +1,16 @@ +import type { SnapshotState } from '../kernel/snapshot.ts'; + +/** + * The one agent-facing explanation for an Android capture that faithfully shows an occluding + * system surface (notification shade, quick settings) instead of app content. Shared by the + * direct snapshot warning and every selector-backed consumer (find/wait/get/is), so the + * disclosure cannot silently drop on one route while surviving on another. + */ +export const ANDROID_SYSTEM_SURFACE_DISCLOSURE = + 'A system surface (notification shade, quick settings, or another system overlay) covers the app, so this snapshot shows that surface. Interact with it or dismiss it (press back or swipe up) to reach app content.'; + +export function systemSurfaceDisclosure( + snapshot: Pick | undefined, +): string | undefined { + return snapshot?.systemSurfaceOnly === true ? ANDROID_SYSTEM_SURFACE_DISCLOSURE : undefined; +}