test: replace the hand-typed Maestro fixture with a generated conformance oracle

.fallowrc.json

@@ -28,6 +28,7 @@
     "examples/test-app/**",
     "scripts/perf/**",
     "scripts/layering/**",
+    "scripts/maestro-conformance/**",
     "apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests.xctestplan",
     "scripts/write-xcuitest-cache-metadata.mjs"
   ],
@@ -37,6 +38,11 @@
       "file": "src/__tests__/test-utils/index.ts",
       "exports": ["*"]
     },
+    {
+      "comment": "Authoritative supported-command list; sole consumer is the conformance oracle (scripts/maestro-conformance, an ignored tooling tree).",
+      "file": "src/compat/maestro/program-ir-command-parser.ts",
+      "exports": ["SUPPORTED_MAESTRO_COMMAND_NAMES"]
+    },
     {
       "file": "src/__tests__/test-utils/device-fixtures.ts",
       "exports": ["LINUX_DEVICE", "ANDROID_TV_DEVICE", "TVOS_SIMULATOR"]

.github/actions/setup-fixture-app/action.yml

@@ -0,0 +1,113 @@
+name: "Setup Fixture App"
+description: "Build (or restore from cache) the examples/test-app fixture app and install it on the booted iOS simulator"
+
+# Any job that needs a controlled app to drive can use this instead of an Apple
+# system app. Building it costs ~22 minutes, so the built .app is cached and the
+# build only runs when its sources or the toolchain actually change.
+#
+# The cache is shared: GitHub caches are per-repository and readable across
+# workflows, and a run can restore caches from its own branch or the default
+# branch. So once a run on main populates it, every workflow gets the hit. The
+# key is computed here from a fixed input list, so all callers agree on it —
+# do not fold caller-specific inputs into it, or the cache stops being shared.
+#
+# Relevant to #320 (move replay coverage off system apps onto a stable fixture).
+
+inputs:
+  runtime-version:
+    description: "iOS runtime version (participates in the cache key)"
+    required: true
+  device-name:
+    description: "Simulator device name used for the build destination"
+    required: false
+    default: "iPhone 17 Pro"
+  install:
+    description: "Install the app onto the booted simulator"
+    required: false
+    default: "true"
+
+outputs:
+  app-path:
+    description: "Path to the built .app bundle"
+    value: ${{ steps.locate.outputs.app-path }}
+  app-id:
+    description: "Bundle identifier read from the built app's Info.plist"
+    value: ${{ steps.locate.outputs.app-id }}
+  cache-hit:
+    description: "Whether the build was restored from cache"
+    value: ${{ steps.cache.outputs.cache-hit }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Resolve Xcode cache key
+      id: xcode
+      shell: bash
+      run: |
+        set -euo pipefail
+        XCODE_KEY="$(xcodebuild -version | tr '\n' ' ' | sed -E 's/[[:space:]]+/ /g; s/[[:space:]]$//' | tr ' ' '-' | tr -cd '[:alnum:]._-')"
+        echo "key=$XCODE_KEY" >> "$GITHUB_OUTPUT"
+
+    - name: Cache fixture app build
+      id: cache
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.3
+      with:
+        path: ${{ github.workspace }}/.tmp/fixture-app
+        # Everything that can change the binary: app sources, native config,
+        # dependency graph, this action's build logic, and the toolchain.
+        # Deliberately caller-independent so the cache is shared across workflows.
+        key: fixture-app-ios-${{ inputs.runtime-version }}-${{ steps.xcode.outputs.key }}-${{ hashFiles('examples/test-app/src/**', 'examples/test-app/app/**', 'examples/test-app/modules/**', 'examples/test-app/app.config.js', 'examples/test-app/package.json', 'examples/test-app/pnpm-lock.yaml', 'examples/test-app/pnpm-workspace.yaml', '.github/actions/setup-fixture-app/action.yml') }}
+
+    - name: Build fixture app
+      if: steps.cache.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        set -euo pipefail
+        # Release embeds the JS bundle, so no Metro server is needed in CI.
+        pnpm test-app:install
+        pnpm --dir examples/test-app exec expo run:ios \
+          --configuration Release \
+          --device "${{ inputs.device-name }}" \
+          --no-bundler
+        # Stash the bundle at a stable path: DerivedData paths are hashed per
+        # project, so the cache needs somewhere deterministic to keep it.
+        APP_PATH="$(find "$HOME/Library/Developer/Xcode/DerivedData" -type d -name '*.app' -path '*Release-iphonesimulator*' | head -1)"
+        if [ -z "$APP_PATH" ]; then
+          echo "::error::Built the fixture app but could not locate its .app bundle to cache."
+          exit 1
+        fi
+        mkdir -p "${{ github.workspace }}/.tmp/fixture-app"
+        rm -rf "${{ github.workspace }}/.tmp/fixture-app/"*.app
+        cp -R "$APP_PATH" "${{ github.workspace }}/.tmp/fixture-app/"
+
+    - name: Locate fixture app
+      id: locate
+      shell: bash
+      run: |
+        set -euo pipefail
+        APP="$(find "${{ github.workspace }}/.tmp/fixture-app" -maxdepth 1 -type d -name '*.app' | head -1)"
+        if [ -z "$APP" ]; then
+          echo "::error::No fixture app bundle at .tmp/fixture-app (restored an empty cache?)."
+          exit 1
+        fi
+        # Read the id from the bundle itself rather than duplicating it here, so
+        # this cannot drift from what was actually built.
+        APP_ID="$(/usr/libexec/PlistBuddy -c 'Print :CFBundleIdentifier' "$APP/Info.plist")"
+        echo "app-path=$APP" >> "$GITHUB_OUTPUT"
+        echo "app-id=$APP_ID" >> "$GITHUB_OUTPUT"
+        echo "fixture app: $(basename "$APP") ($APP_ID)"
+
+    - name: Install fixture app
+      if: inputs.install == 'true'
+      shell: bash
+      run: |
+        set -euo pipefail
+        xcrun simctl install booted "${{ steps.locate.outputs.app-path }}"
+        # Fail loudly rather than let a caller drive a device with no app: a
+        # missing app produces scenarios that fail for the wrong reason, or pass
+        # while proving nothing.
+        if ! xcrun simctl get_app_container booted "${{ steps.locate.outputs.app-id }}" >/dev/null 2>&1; then
+          echo "::error::Fixture app ${{ steps.locate.outputs.app-id }} is not installed on the booted simulator."
+          exit 1
+        fi
+        echo "installed ${{ steps.locate.outputs.app-id }}"

.github/workflows/ci.yml

@@ -129,6 +129,28 @@ jobs:
       - name: Check affected-selector model
         run: node --experimental-strip-types --test scripts/check-affected/model.test.ts scripts/check-affected/run.test.ts
 
+  maestro-conformance:
+    name: Maestro Conformance Oracle
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      # Unlike the layering/affected guards, this job DOES install deps: the
+      # verifier parses corpus flows with the live engine, and the Maestro parser
+      # imports the `yaml` package. Keep install-deps enabled.
+      - name: Setup toolchain
+        uses: ./.github/actions/setup-node-pnpm
+
+      # Layers 1-2 of the conformance oracle: replay the JVM-generated fixtures
+      # against the live engine. Deterministic and Java-free — the generated
+      # fixtures are checked in and only regenerated on an upstream-pin bump. The
+      # device-backed layer 3 runs on the scheduled conformance-differential
+      # workflow. See scripts/maestro-conformance/README.md.
+      - name: Verify Maestro conformance fixtures
+        run: pnpm maestro:conformance
+
   packaged-cli-node-22-12:
     name: Packaged CLI Node 22.12
     runs-on: ubuntu-latest

.github/workflows/conformance-differential.yml

@@ -0,0 +1,145 @@
+name: Conformance Differential
+
+# Layer 3 of the Maestro conformance oracle: run a small set of flows through
+# BOTH real Maestro and agent-device on a live device and compare app-observable
+# outcomes, plus assert engine-side timing invariants over agent-device's own
+# replay trace. This is the only layer that can catch settle-loop ordering, which
+# has no reflectable upstream constant.
+#
+# Scheduled: this path is proven end-to-end (run 29504440599 executed both
+# engines against the real fixture app). Scenarios that currently diverge are
+# DECLARED via `knownDivergence` in differential/scenarios.ts with a tracking
+# issue, so the schedule stays green on known gaps and fails only on undeclared
+# ones — the same contract layer 1 has. A declaration that stops reproducing also
+# fails, so a fix cannot land while leaving the oracle blind.
+
+on:
+  schedule:
+    - cron: "0 5 * * *"
+  workflow_dispatch:
+    inputs:
+      only:
+        description: "Single scenario id to run (default: all)"
+        required: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  AGENT_DEVICE_CLI: "--experimental-strip-types src/bin.ts"
+  DIFFERENTIAL_ONLY: ${{ github.event.inputs.only || '' }}
+  # CI should not phone home, and it keeps `maestro --version` to just the
+  # version instead of prefixing an analytics notice.
+  MAESTRO_CLI_NO_ANALYTICS: "1"
+
+jobs:
+  differential-ios:
+    name: iOS Conformance Differential
+    runs-on: macos-26
+    timeout-minutes: 90
+    env:
+      IOS_RUNTIME_VERSION: "26.2"
+      AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state
+      AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup toolchain
+        uses: ./.github/actions/setup-node-pnpm
+
+      - name: Setup Apple replay
+        uses: ./.github/actions/setup-apple-replay
+        with:
+          derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
+          cache-key-prefix: ios-runner-prebuilt
+          cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}
+          build-command: sh ./scripts/build-xcuitest-apple.sh
+          xcuitest-platform: ios
+
+      - name: Boot simulator
+        uses: ./.github/actions/boot-ios-test-simulator
+        with:
+          runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
+          preferred-device-name: iPhone 17 Pro
+
+      # Builds or restores the cached fixture app and installs it. Shared with any
+      # other job that needs a controlled app to drive (see #320) — the cache is
+      # repo-wide, so whichever workflow builds it first pays the ~22 minutes and
+      # the rest get a hit.
+      - name: Setup fixture app
+        id: fixture-app
+        uses: ./.github/actions/setup-fixture-app
+        with:
+          runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
+          device-name: iPhone 17 Pro
+
+      # The action guarantees *an* app is installed; this asserts it is the one
+      # the scenarios actually target, so a fixture rename cannot leave the
+      # differential driving the wrong app.
+      - name: Verify the installed app is the one the scenarios target
+        run: |
+          set -euo pipefail
+          EXPECTED=$(node --experimental-strip-types -e \
+            "import('./scripts/maestro-conformance/differential/scenarios.ts').then(m => console.log(m.DIFFERENTIAL_APP_ID))")
+          ACTUAL="${{ steps.fixture-app.outputs.app-id }}"
+          echo "expected=$EXPECTED actual=$ACTUAL (cache-hit=${{ steps.fixture-app.outputs.cache-hit }})"
+          if [ "$ACTUAL" != "$EXPECTED" ]; then
+            echo "::error::Installed fixture app is $ACTUAL but the scenarios target $EXPECTED; they would fail vacuously."
+            exit 1
+          fi
+
+      # Pin the Maestro CLI to the SAME version the oracle pins for layers 1-2,
+      # read from pinned-upstream.json so the two can never drift apart. The
+      # installer honours MAESTRO_VERSION; assert what actually landed.
+      - name: Install pinned Maestro CLI
+        run: |
+          set -euo pipefail
+          MAESTRO_VERSION=$(node -p "require('./scripts/maestro-conformance/pinned-upstream.json').version")
+          export MAESTRO_VERSION
+          echo "Pinning Maestro CLI to $MAESTRO_VERSION (from pinned-upstream.json)"
+          curl -fsSL "https://get.maestro.mobile.dev" | bash
+          echo "$HOME/.maestro/bin" >> "$GITHUB_PATH"
+
+      - name: Verify the Maestro CLI matches the oracle pin
+        run: |
+          set -euo pipefail
+          EXPECTED=$(node -p "require('./scripts/maestro-conformance/pinned-upstream.json').version")
+          # `maestro --version` can print notices (e.g. the analytics banner)
+          # around the version, so match the semver line rather than the whole
+          # output. Empty ACTUAL fails the comparison below, as it should.
+          ACTUAL=$(maestro --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | tail -1 || true)
+          echo "expected=$EXPECTED actual=$ACTUAL"
+          if [ "$ACTUAL" != "$EXPECTED" ]; then
+            echo "::error::Layer-3 Maestro CLI is $ACTUAL but the oracle pins $EXPECTED. A differential against a different Maestro proves nothing about the pinned behavior."
+            exit 1
+          fi
+
+      # --trace-root lets the runner find agent-device's replay-timing.ndjson and
+      # assert engine-side invariants (e.g. the settle loop latches instead of
+      # burning its budget) — outcome parity alone cannot see that.
+      - name: Run differential
+        run: |
+          node --experimental-strip-types scripts/maestro-conformance/differential/run.ts \
+            --platform ios \
+            --out-dir "${{ github.workspace }}/.tmp/conformance-differential" \
+            --trace-root "${{ github.workspace }}/.agent-device" \
+            ${DIFFERENTIAL_ONLY:+--only "$DIFFERENTIAL_ONLY"}
+
+      # Keep the trace, not just the verdict: the engine-side invariants are
+      # computed FROM replay-timing.ndjson, so without it a report saying
+      # "invariant violated: tapRetries was 0" cannot be audited or debugged
+      # after the runner is gone.
+      - name: Upload report and trace
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: conformance-differential-ios
+          path: |
+            .tmp/conformance-differential/differential-report.json
+            .agent-device/**/replay-timing.ndjson
+          if-no-files-found: ignore

.github/workflows/conformance-regenerate.yml

@@ -0,0 +1,66 @@
+name: Conformance Regenerate Verify
+
+# Re-derives the layer-1/layer-2 fixtures from the pinned upstream Maestro
+# artifacts and fails if the checked-in files differ by a single byte.
+#
+# Why this exists as its own job: per-PR CI must stay Java-free (ADR 0015), so it
+# can only recompute the fixtures' content seal. That seal is tamper-EVIDENT — it
+# stops accidental or casual hand-editing — but someone could edit a capture and
+# recompute the hash. This job closes that hole for real: forgery cannot survive
+# an actual re-derivation against the pinned jars. It is what makes "generated
+# from upstream" an enforced property rather than a claim in a README.
+#
+# Scheduled + manual only. It needs a JDK, Gradle, and network access to Maven
+# Central, none of which belong on the per-PR path.
+
+on:
+  schedule:
+    - cron: "0 6 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  regenerate-verify:
+    name: Regenerate & Verify Fixtures
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      # regenerate.mjs itself needs only Node builtins, but the verify step below
+      # parses corpus flows with the live engine, which imports the `yaml`
+      # package — so deps must be installed here.
+      - name: Setup toolchain
+        uses: ./.github/actions/setup-node-pnpm
+
+      - name: Setup JDK
+        uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
+        with:
+          distribution: temurin
+          java-version: "17"
+
+      # Verifies the pinned jar SHA-256s against pinned-upstream.json, runs the
+      # harness over the corpus, and rewrites fixtures/ + corpus/manifest.json.
+      - name: Regenerate fixtures from pinned upstream
+        run: node scripts/maestro-conformance/regenerate.mjs
+
+      - name: Fail if regeneration changed anything
+        run: |
+          if ! git diff --exit-code -- scripts/maestro-conformance/fixtures scripts/maestro-conformance/corpus/manifest.json; then
+            echo "::error::Checked-in conformance fixtures do not match a fresh regeneration from the pinned upstream artifacts."
+            echo "The fixtures are generated: run 'pnpm maestro:conformance:regenerate' and commit the result."
+            exit 1
+          fi
+          echo "Fixtures are byte-identical to a fresh regeneration from Maestro ${{ github.sha }}."
+
+      # The seal check also runs per-PR, but assert it here too so a regeneration
+      # that produced an unsealed/mis-sealed fixture cannot pass silently.
+      - name: Verify fixture seals and conformance
+        run: node --experimental-strip-types --test scripts/maestro-conformance/verify.test.ts