Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions src/commands/capture/runtime/snapshot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,27 @@ test('runtime snapshot emits filtered Android guidance from backend analysis', a
]);
});

test('runtime snapshot renders the system-surface disclosure from Android annotations', async () => {
const device = createSnapshotOnlyDevice({
nodes: [
{ ref: 'e1', index: 0, depth: 0, type: 'FrameLayout', label: 'Quick settings' },
{ ref: 'e2', index: 1, depth: 1, parentIndex: 0, type: 'Switch', label: 'Internet' },
],
truncated: false,
backend: 'android',
androidSnapshot: {
backend: 'android-helper',
systemSurfaceOnly: true,
},
});

const result = await device.capture.snapshot({ session: 'default' });

assert.equal(result.warnings?.length, 1);
assert.match(String(result.warnings?.[0]), /system surface \(notification shade, quick settings/);
assert.match(String(result.warnings?.[0]), /press back or swipe up/);
});

test('runtime snapshot warns when iOS interactive output is root-only', async () => {
const device = createSnapshotOnlyDevice({
nodes: [{ ref: 'e1', index: 0, depth: 0, type: 'Application' }],
Expand Down
11 changes: 11 additions & 0 deletions src/commands/capture/runtime/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import type {
SnapshotVisibility,
} from '../../../kernel/snapshot.ts';
import { buildSnapshotVisibility } from '../../../snapshot/snapshot-visibility.ts';
import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '../../../snapshot/system-surface-disclosure.ts';
import { formatReactNativeOverlayWarning } from '../../react-native/overlay.ts';
import {
buildUnchangedSnapshotMetadata,
Expand Down Expand Up @@ -243,13 +244,23 @@ function buildSnapshotWarnings(params: {
const reactNativeOverlayWarning = formatReactNativeOverlayWarning(params.snapshot.nodes);
if (reactNativeOverlayWarning) warnings.push(reactNativeOverlayWarning);

const systemSurfaceWarning = formatAndroidSystemSurfaceWarning(params.annotations);
if (systemSurfaceWarning) warnings.push(systemSurfaceWarning);

const recentDropWarning = formatRecentSnapshotDropWarning(params);
if (recentDropWarning) warnings.push(recentDropWarning);

warnings.push(...formatFreshnessWarnings(params.annotations.freshness, params.snapshot.backend));
return Array.from(new Set(warnings));
}

function formatAndroidSystemSurfaceWarning(
annotations: SnapshotCaptureAnnotations,
): string | undefined {
if (annotations.androidSnapshot?.systemSurfaceOnly !== true) return undefined;
return ANDROID_SYSTEM_SURFACE_DISCLOSURE;
}

function buildSparseIosInteractiveWarnings(params: {
snapshot: SnapshotState;
options: SnapshotCommandOptions;
Expand Down
69 changes: 69 additions & 0 deletions src/contracts/android-system-chrome.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/**
* Android status-bar/navigation-bar chrome markers, shared between the settle-chrome
* classifier (`core/snapshot-chrome.ts`, #1198) and the helper content classifier
* (`platforms/android/snapshot-content-recovery.ts`). SystemUI hosts BOTH persistent chrome
* and actionable overlays (volume panel, media/output pickers, notification shade, quick
* settings), so chrome is never a package-level fact: only these status/nav-bar marker
* resource-ids classify as chrome; every other systemui surface is real content.
*/
export const ANDROID_SYSTEM_CHROME_PACKAGE = 'com.android.systemui';

const ANDROID_SYSTEM_CHROME_MARKER_PREFIXES = [
'com.android.systemui:id/status_bar',
'com.android.systemui:id/navigation_bar',
] as const;

/**
* Surviving status-bar/nav-bar LEAF ids (#1251). The non-raw Android walk
* (`walkUiHierarchyNode` in `platforms/android/ui-hierarchy.ts`) drops
* unlabeled/unidentified structural nodes via `shouldIncludeStructuralAndroidNode`,
* re-parenting their children upward — and that silently swallows every
* `status_bar*`/`navigation_bar*` WRAPPER node, i.e. the only nodes the prefix
* check above matches. A non-raw capture is left with just their labeled/
* identified LEAVES (clock, battery, wifi/mobile icons, nav buttons), whose
* own resource-ids carry no `status_bar`/`navigation_bar` prefix, so the run
* loses its marker and `collectAndroidSystemChromeRunIndexes` stops dropping
* it (verified against a real `--raw` vs. default capture pair of the same
* screen). Recognize those leaves directly, by EXACT id — not prefix, to stay
* tight: nothing here should ever swallow an actionable systemui overlay like
* the volume dialog or a media/output picker, which live under unrelated ids.
* `--raw` keeps the wrapper markers, so the prefix check above stays
* load-bearing there.
*
* A more robust fix would thread the AOSP window-type constants
* (`TYPE_STATUS_BAR` = 2000, `TYPE_NAVIGATION_BAR` = 2019) already parsed in
* `readNodeAttributes` (ui-hierarchy.ts) through to the output `SnapshotNode`
* and key off that instead of resource-ids — deferred until the id-based
* approach here proves insufficient.
*/
const ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS: ReadonlySet<string> = new Set(
[
// status bar
'clock',
'battery',
'statusIcons',
'notificationIcons',
'notification_icon_area',
'system_icons',
'cutout_space_view',
'mobile_signal',
'mobile_combo',
'mobile_group',
'wifi_signal',
'wifi_combo',
'wifi_group',
'start_side_notif_and_chip_container',
// nav bar
'back',
'home',
'recent_apps',
'home_handle',
].map((leaf) => `${ANDROID_SYSTEM_CHROME_PACKAGE}:id/${leaf}`),
);

/** True when the resource-id marks Android status-bar or navigation-bar chrome. */
export function isAndroidSystemChromeResourceId(resourceId: string | null | undefined): boolean {
const identifier = resourceId ?? '';
if (ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS.has(identifier)) return true;
return ANDROID_SYSTEM_CHROME_MARKER_PREFIXES.some((prefix) => identifier.startsWith(prefix));
}
66 changes: 8 additions & 58 deletions src/core/snapshot-chrome.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
import {
ANDROID_SYSTEM_CHROME_PACKAGE,
isAndroidSystemChromeResourceId,
} from '../contracts/android-system-chrome.ts';
import type { SnapshotNode } from '../kernel/snapshot.ts';
import { isAndroidInputMethodSnapshotNode } from '../snapshot/android-input-method-overlays.ts';
import { normalizeType } from '../utils/text-surface.ts';
Expand Down Expand Up @@ -205,65 +209,11 @@ function collectSubtreeIndexes(
// kept. Marker set live-verified on the emulator: the status-bar window
// carries `status_bar*` ids throughout while the VolumeDialog window carries
// only `volume_dialog*` ids (`input_method_nav*` bars are IME-owned and
// handled by the IME tier).
const ANDROID_SYSTEM_CHROME_PACKAGE = 'com.android.systemui';
const ANDROID_SYSTEM_CHROME_MARKER_PREFIXES = [
'com.android.systemui:id/status_bar',
'com.android.systemui:id/navigation_bar',
];

/**
* Surviving status-bar/nav-bar LEAF ids (#1251). The non-raw Android walk
* (`walkUiHierarchyNode` in `platforms/android/ui-hierarchy.ts`) drops
* unlabeled/unidentified structural nodes via `shouldIncludeStructuralAndroidNode`,
* re-parenting their children upward — and that silently swallows every
* `status_bar*`/`navigation_bar*` WRAPPER node, i.e. the only nodes the prefix
* check above matches. A non-raw capture is left with just their labeled/
* identified LEAVES (clock, battery, wifi/mobile icons, nav buttons), whose
* own resource-ids carry no `status_bar`/`navigation_bar` prefix, so the run
* loses its marker and `collectAndroidSystemChromeRunIndexes` stops dropping
* it (verified against a real `--raw` vs. default capture pair of the same
* screen). Recognize those leaves directly, by EXACT id — not prefix, to stay
* tight: nothing here should ever swallow an actionable systemui overlay like
* the volume dialog or a media/output picker, which live under unrelated ids.
* `--raw` keeps the wrapper markers, so the prefix check above stays
* load-bearing there.
*
* A more robust fix would thread the AOSP window-type constants
* (`TYPE_STATUS_BAR` = 2000, `TYPE_NAVIGATION_BAR` = 2019) already parsed in
* `readNodeAttributes` (ui-hierarchy.ts) through to the output `SnapshotNode`
* and key off that instead of resource-ids — deferred until the id-based
* approach here proves insufficient.
*/
const ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS = new Set(
[
// status bar
'clock',
'battery',
'statusIcons',
'notificationIcons',
'notification_icon_area',
'system_icons',
'cutout_space_view',
'mobile_signal',
'mobile_combo',
'mobile_group',
'wifi_signal',
'wifi_combo',
'wifi_group',
'start_side_notif_and_chip_container',
// nav bar
'back',
'home',
'recent_apps',
'home_handle',
].map((leaf) => `${ANDROID_SYSTEM_CHROME_PACKAGE}:id/${leaf}`),
);

// handled by the IME tier). The marker constants and the resource-id-level
// predicate live in `contracts/android-system-chrome.ts` so the Android
// helper content classifier reuses the same classification.
function hasAndroidSystemChromeMarker(node: SnapshotNode): boolean {
const identifier = node.identifier ?? '';
if (ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS.has(identifier)) return true;
return ANDROID_SYSTEM_CHROME_MARKER_PREFIXES.some((prefix) => identifier.startsWith(prefix));
return isAndroidSystemChromeResourceId(node.identifier ?? '');
}

/**
Expand Down
Loading
Loading