diff --git a/.fallowrc.json b/.fallowrc.json index 49c6b00c0..c61ec0d63 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -28,6 +28,7 @@ "examples/test-app/**", "scripts/perf/**", "scripts/layering/**", + "scripts/maestro-conformance/**", "apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests.xctestplan", "scripts/write-xcuitest-cache-metadata.mjs" ], @@ -37,6 +38,11 @@ "file": "src/__tests__/test-utils/index.ts", "exports": ["*"] }, + { + "comment": "Authoritative supported-command list; sole consumer is the conformance oracle (scripts/maestro-conformance, an ignored tooling tree).", + "file": "src/compat/maestro/program-ir-command-parser.ts", + "exports": ["SUPPORTED_MAESTRO_COMMAND_NAMES"] + }, { "file": "src/__tests__/test-utils/device-fixtures.ts", "exports": ["LINUX_DEVICE", "ANDROID_TV_DEVICE", "TVOS_SIMULATOR"] diff --git a/.github/actions/setup-fixture-app/action.yml b/.github/actions/setup-fixture-app/action.yml new file mode 100644 index 000000000..9a2cfc227 --- /dev/null +++ b/.github/actions/setup-fixture-app/action.yml @@ -0,0 +1,113 @@ +name: "Setup Fixture App" +description: "Build (or restore from cache) the examples/test-app fixture app and install it on the booted iOS simulator" + +# Any job that needs a controlled app to drive can use this instead of an Apple +# system app. Building it costs ~22 minutes, so the built .app is cached and the +# build only runs when its sources or the toolchain actually change. +# +# The cache is shared: GitHub caches are per-repository and readable across +# workflows, and a run can restore caches from its own branch or the default +# branch. So once a run on main populates it, every workflow gets the hit. The +# key is computed here from a fixed input list, so all callers agree on it — +# do not fold caller-specific inputs into it, or the cache stops being shared. +# +# Relevant to #320 (move replay coverage off system apps onto a stable fixture). + +inputs: + runtime-version: + description: "iOS runtime version (participates in the cache key)" + required: true + device-name: + description: "Simulator device name used for the build destination" + required: false + default: "iPhone 17 Pro" + install: + description: "Install the app onto the booted simulator" + required: false + default: "true" + +outputs: + app-path: + description: "Path to the built .app bundle" + value: ${{ steps.locate.outputs.app-path }} + app-id: + description: "Bundle identifier read from the built app's Info.plist" + value: ${{ steps.locate.outputs.app-id }} + cache-hit: + description: "Whether the build was restored from cache" + value: ${{ steps.cache.outputs.cache-hit }} + +runs: + using: "composite" + steps: + - name: Resolve Xcode cache key + id: xcode + shell: bash + run: | + set -euo pipefail + XCODE_KEY="$(xcodebuild -version | tr '\n' ' ' | sed -E 's/[[:space:]]+/ /g; s/[[:space:]]$//' | tr ' ' '-' | tr -cd '[:alnum:]._-')" + echo "key=$XCODE_KEY" >> "$GITHUB_OUTPUT" + + - name: Cache fixture app build + id: cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.2.3 + with: + path: ${{ github.workspace }}/.tmp/fixture-app + # Everything that can change the binary: app sources, native config, + # dependency graph, this action's build logic, and the toolchain. + # Deliberately caller-independent so the cache is shared across workflows. + key: fixture-app-ios-${{ inputs.runtime-version }}-${{ steps.xcode.outputs.key }}-${{ hashFiles('examples/test-app/src/**', 'examples/test-app/app/**', 'examples/test-app/modules/**', 'examples/test-app/app.config.js', 'examples/test-app/package.json', 'examples/test-app/pnpm-lock.yaml', 'examples/test-app/pnpm-workspace.yaml', '.github/actions/setup-fixture-app/action.yml') }} + + - name: Build fixture app + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + run: | + set -euo pipefail + # Release embeds the JS bundle, so no Metro server is needed in CI. + pnpm test-app:install + pnpm --dir examples/test-app exec expo run:ios \ + --configuration Release \ + --device "${{ inputs.device-name }}" \ + --no-bundler + # Stash the bundle at a stable path: DerivedData paths are hashed per + # project, so the cache needs somewhere deterministic to keep it. + APP_PATH="$(find "$HOME/Library/Developer/Xcode/DerivedData" -type d -name '*.app' -path '*Release-iphonesimulator*' | head -1)" + if [ -z "$APP_PATH" ]; then + echo "::error::Built the fixture app but could not locate its .app bundle to cache." + exit 1 + fi + mkdir -p "${{ github.workspace }}/.tmp/fixture-app" + rm -rf "${{ github.workspace }}/.tmp/fixture-app/"*.app + cp -R "$APP_PATH" "${{ github.workspace }}/.tmp/fixture-app/" + + - name: Locate fixture app + id: locate + shell: bash + run: | + set -euo pipefail + APP="$(find "${{ github.workspace }}/.tmp/fixture-app" -maxdepth 1 -type d -name '*.app' | head -1)" + if [ -z "$APP" ]; then + echo "::error::No fixture app bundle at .tmp/fixture-app (restored an empty cache?)." + exit 1 + fi + # Read the id from the bundle itself rather than duplicating it here, so + # this cannot drift from what was actually built. + APP_ID="$(/usr/libexec/PlistBuddy -c 'Print :CFBundleIdentifier' "$APP/Info.plist")" + echo "app-path=$APP" >> "$GITHUB_OUTPUT" + echo "app-id=$APP_ID" >> "$GITHUB_OUTPUT" + echo "fixture app: $(basename "$APP") ($APP_ID)" + + - name: Install fixture app + if: inputs.install == 'true' + shell: bash + run: | + set -euo pipefail + xcrun simctl install booted "${{ steps.locate.outputs.app-path }}" + # Fail loudly rather than let a caller drive a device with no app: a + # missing app produces scenarios that fail for the wrong reason, or pass + # while proving nothing. + if ! xcrun simctl get_app_container booted "${{ steps.locate.outputs.app-id }}" >/dev/null 2>&1; then + echo "::error::Fixture app ${{ steps.locate.outputs.app-id }} is not installed on the booted simulator." + exit 1 + fi + echo "installed ${{ steps.locate.outputs.app-id }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9403afef9..79976c838 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,6 +129,28 @@ jobs: - name: Check affected-selector model run: node --experimental-strip-types --test scripts/check-affected/model.test.ts scripts/check-affected/run.test.ts + maestro-conformance: + name: Maestro Conformance Oracle + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # Unlike the layering/affected guards, this job DOES install deps: the + # verifier parses corpus flows with the live engine, and the Maestro parser + # imports the `yaml` package. Keep install-deps enabled. + - name: Setup toolchain + uses: ./.github/actions/setup-node-pnpm + + # Layers 1-2 of the conformance oracle: replay the JVM-generated fixtures + # against the live engine. Deterministic and Java-free — the generated + # fixtures are checked in and only regenerated on an upstream-pin bump. The + # device-backed layer 3 runs on the scheduled conformance-differential + # workflow. See scripts/maestro-conformance/README.md. + - name: Verify Maestro conformance fixtures + run: pnpm maestro:conformance + packaged-cli-node-22-12: name: Packaged CLI Node 22.12 runs-on: ubuntu-latest diff --git a/.github/workflows/conformance-differential.yml b/.github/workflows/conformance-differential.yml new file mode 100644 index 000000000..7098cfc86 --- /dev/null +++ b/.github/workflows/conformance-differential.yml @@ -0,0 +1,145 @@ +name: Conformance Differential + +# Layer 3 of the Maestro conformance oracle: run a small set of flows through +# BOTH real Maestro and agent-device on a live device and compare app-observable +# outcomes, plus assert engine-side timing invariants over agent-device's own +# replay trace. This is the only layer that can catch settle-loop ordering, which +# has no reflectable upstream constant. +# +# Scheduled: this path is proven end-to-end (run 29504440599 executed both +# engines against the real fixture app). Scenarios that currently diverge are +# DECLARED via `knownDivergence` in differential/scenarios.ts with a tracking +# issue, so the schedule stays green on known gaps and fails only on undeclared +# ones — the same contract layer 1 has. A declaration that stops reproducing also +# fails, so a fix cannot land while leaving the oracle blind. + +on: + schedule: + - cron: "0 5 * * *" + workflow_dispatch: + inputs: + only: + description: "Single scenario id to run (default: all)" + required: false + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + AGENT_DEVICE_CLI: "--experimental-strip-types src/bin.ts" + DIFFERENTIAL_ONLY: ${{ github.event.inputs.only || '' }} + # CI should not phone home, and it keeps `maestro --version` to just the + # version instead of prefixing an analytics notice. + MAESTRO_CLI_NO_ANALYTICS: "1" + +jobs: + differential-ios: + name: iOS Conformance Differential + runs-on: macos-26 + timeout-minutes: 90 + env: + IOS_RUNTIME_VERSION: "26.2" + AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state + AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup toolchain + uses: ./.github/actions/setup-node-pnpm + + - name: Setup Apple replay + uses: ./.github/actions/setup-apple-replay + with: + derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} + cache-key-prefix: ios-runner-prebuilt + cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }} + build-command: sh ./scripts/build-xcuitest-apple.sh + xcuitest-platform: ios + + - name: Boot simulator + uses: ./.github/actions/boot-ios-test-simulator + with: + runtime-version: ${{ env.IOS_RUNTIME_VERSION }} + preferred-device-name: iPhone 17 Pro + + # Builds or restores the cached fixture app and installs it. Shared with any + # other job that needs a controlled app to drive (see #320) — the cache is + # repo-wide, so whichever workflow builds it first pays the ~22 minutes and + # the rest get a hit. + - name: Setup fixture app + id: fixture-app + uses: ./.github/actions/setup-fixture-app + with: + runtime-version: ${{ env.IOS_RUNTIME_VERSION }} + device-name: iPhone 17 Pro + + # The action guarantees *an* app is installed; this asserts it is the one + # the scenarios actually target, so a fixture rename cannot leave the + # differential driving the wrong app. + - name: Verify the installed app is the one the scenarios target + run: | + set -euo pipefail + EXPECTED=$(node --experimental-strip-types -e \ + "import('./scripts/maestro-conformance/differential/scenarios.ts').then(m => console.log(m.DIFFERENTIAL_APP_ID))") + ACTUAL="${{ steps.fixture-app.outputs.app-id }}" + echo "expected=$EXPECTED actual=$ACTUAL (cache-hit=${{ steps.fixture-app.outputs.cache-hit }})" + if [ "$ACTUAL" != "$EXPECTED" ]; then + echo "::error::Installed fixture app is $ACTUAL but the scenarios target $EXPECTED; they would fail vacuously." + exit 1 + fi + + # Pin the Maestro CLI to the SAME version the oracle pins for layers 1-2, + # read from pinned-upstream.json so the two can never drift apart. The + # installer honours MAESTRO_VERSION; assert what actually landed. + - name: Install pinned Maestro CLI + run: | + set -euo pipefail + MAESTRO_VERSION=$(node -p "require('./scripts/maestro-conformance/pinned-upstream.json').version") + export MAESTRO_VERSION + echo "Pinning Maestro CLI to $MAESTRO_VERSION (from pinned-upstream.json)" + curl -fsSL "https://get.maestro.mobile.dev" | bash + echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" + + - name: Verify the Maestro CLI matches the oracle pin + run: | + set -euo pipefail + EXPECTED=$(node -p "require('./scripts/maestro-conformance/pinned-upstream.json').version") + # `maestro --version` can print notices (e.g. the analytics banner) + # around the version, so match the semver line rather than the whole + # output. Empty ACTUAL fails the comparison below, as it should. + ACTUAL=$(maestro --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | tail -1 || true) + echo "expected=$EXPECTED actual=$ACTUAL" + if [ "$ACTUAL" != "$EXPECTED" ]; then + echo "::error::Layer-3 Maestro CLI is $ACTUAL but the oracle pins $EXPECTED. A differential against a different Maestro proves nothing about the pinned behavior." + exit 1 + fi + + # --trace-root lets the runner find agent-device's replay-timing.ndjson and + # assert engine-side invariants (e.g. the settle loop latches instead of + # burning its budget) — outcome parity alone cannot see that. + - name: Run differential + run: | + node --experimental-strip-types scripts/maestro-conformance/differential/run.ts \ + --platform ios \ + --out-dir "${{ github.workspace }}/.tmp/conformance-differential" \ + --trace-root "${{ github.workspace }}/.agent-device" \ + ${DIFFERENTIAL_ONLY:+--only "$DIFFERENTIAL_ONLY"} + + # Keep the trace, not just the verdict: the engine-side invariants are + # computed FROM replay-timing.ndjson, so without it a report saying + # "invariant violated: tapRetries was 0" cannot be audited or debugged + # after the runner is gone. + - name: Upload report and trace + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: conformance-differential-ios + path: | + .tmp/conformance-differential/differential-report.json + .agent-device/**/replay-timing.ndjson + if-no-files-found: ignore diff --git a/.github/workflows/conformance-regenerate.yml b/.github/workflows/conformance-regenerate.yml new file mode 100644 index 000000000..9b0f397c3 --- /dev/null +++ b/.github/workflows/conformance-regenerate.yml @@ -0,0 +1,66 @@ +name: Conformance Regenerate Verify + +# Re-derives the layer-1/layer-2 fixtures from the pinned upstream Maestro +# artifacts and fails if the checked-in files differ by a single byte. +# +# Why this exists as its own job: per-PR CI must stay Java-free (ADR 0015), so it +# can only recompute the fixtures' content seal. That seal is tamper-EVIDENT — it +# stops accidental or casual hand-editing — but someone could edit a capture and +# recompute the hash. This job closes that hole for real: forgery cannot survive +# an actual re-derivation against the pinned jars. It is what makes "generated +# from upstream" an enforced property rather than a claim in a README. +# +# Scheduled + manual only. It needs a JDK, Gradle, and network access to Maven +# Central, none of which belong on the per-PR path. + +on: + schedule: + - cron: "0 6 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + regenerate-verify: + name: Regenerate & Verify Fixtures + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # regenerate.mjs itself needs only Node builtins, but the verify step below + # parses corpus flows with the live engine, which imports the `yaml` + # package — so deps must be installed here. + - name: Setup toolchain + uses: ./.github/actions/setup-node-pnpm + + - name: Setup JDK + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: temurin + java-version: "17" + + # Verifies the pinned jar SHA-256s against pinned-upstream.json, runs the + # harness over the corpus, and rewrites fixtures/ + corpus/manifest.json. + - name: Regenerate fixtures from pinned upstream + run: node scripts/maestro-conformance/regenerate.mjs + + - name: Fail if regeneration changed anything + run: | + if ! git diff --exit-code -- scripts/maestro-conformance/fixtures scripts/maestro-conformance/corpus/manifest.json; then + echo "::error::Checked-in conformance fixtures do not match a fresh regeneration from the pinned upstream artifacts." + echo "The fixtures are generated: run 'pnpm maestro:conformance:regenerate' and commit the result." + exit 1 + fi + echo "Fixtures are byte-identical to a fresh regeneration from Maestro ${{ github.sha }}." + + # The seal check also runs per-PR, but assert it here too so a regeneration + # that produced an unsealed/mis-sealed fixture cannot pass silently. + - name: Verify fixture seals and conformance + run: node --experimental-strip-types --test scripts/maestro-conformance/verify.test.ts diff --git a/docs/adr/0015-direct-maestro-engine.md b/docs/adr/0015-direct-maestro-engine.md index 3dc12dfa2..24afa91a1 100644 --- a/docs/adr/0015-direct-maestro-engine.md +++ b/docs/adr/0015-direct-maestro-engine.md @@ -122,13 +122,38 @@ mutation boundary; a later mutating command still verifies hierarchy stability b comparison captures use the persistent runner's screenshot surface so both frames come from one warmed transport and avoid simulator screenshot setup between polls. -Upstream Maestro is a version-pinned development reference, not a production dependency. The current -opt-in conformance script compares a small checked-in parser corpus against manually captured Maestro -2.5.1 output; it is not a CI gate and does not prove runtime parity. The accepted follow-up is a -three-layer oracle: parser normalization generated from the pinned upstream corpus, semantic vectors -generated by a JVM harness that calls upstream geometry/default/retry/timing code, and a small set of -app-observable differential scenarios. Deterministic generated layers will run in CI without requiring -Java during normal checks. +Upstream Maestro is a version-pinned development reference, not a production dependency. Conformance is +enforced by the three-layer oracle in `scripts/maestro-conformance` (issue #1274), which replaced the +original hand-typed parser fixture. Layer 1 drives the pinned `maestro-orchestra` YAML parser +(`dev.mobile:maestro-orchestra:2.5.1`, SHA-verified at regeneration) over a corpus of vendored upstream +flows plus targeted bug-class flows, capturing each parse; the deterministic verifier replays that +capture against our engine and classifies every flow as identical / both-reject / we-reject / mismatch, +requiring each non-identical outcome to be a declared, on-the-record divergence. Layer 2 reads upstream +constants (retry cap, animation-wait threshold/timeout, erase cap) and parser-observed defaults (swipe +duration) straight from the pinned bytecode and cross-checks each against the live `MAESTRO_COMPATIBILITY_PRESETS` +constant. Both generated layers are checked in and verified in CI via `node --test` with no Java. Because +per-PR CI cannot re-derive them, each fixture carries a `contentHash` seal that the verifier recomputes, +so a hand edit to a captured command or constant fails the build; the scheduled `conformance-regenerate` +job then re-runs the harness against the pinned jars and fails on any byte difference, which is what makes +"generated from upstream" enforced rather than documented. Layer 3 runs a small set of app-observable +differential scenarios (settle ordering, tap retries, optional warned-vs-failed) through both engines +against the real fixture app, pinning the Maestro CLI to the same version as layers 1-2; it is scheduled, +never per-PR. Layer 3 carries the same declared-divergence contract as layer 1: a scenario that currently +fails declares a `knownDivergence` with a required tracking issue, the schedule stays green on that known +gap, and only undeclared divergences fail. A declaration that stops reproducing also fails, so the fix PR +must remove it and the differential becomes the acceptance test for its own findings. This exists because +the instrument must not be blocked on repairing what it just measured. The four #1217 regressions (percent +rounding/rejection, target-swipe direction default, retry-cap semantics, settle-loop ordering) each have a +dedicated fixture. Percent truncation's runtime half is pinned by a pure unit test rather than a device +scenario, because a one-pixel delta is not app-observable. Bug class 4's detector is an engine-side timing +invariant, unit-tested against synthetic traces, that executes on the device path — currently blocked by a +declared `scrollUntilVisible` divergence (#1299) that the differential itself found on its first working run. + +Documented intentional deviations are encoded as expected-divergence entries rather than silent +mismatches: the hierarchy-vs-screenshot animation wait (constants match, mechanism differs), the +horizontal-swipe in-page presets, condition-poll-as-stabilization, ambiguity strictness under `optional`, +and the omitted iOS 3s pre-tap static gate (`IOSDriver.SCREEN_SETTLE_TIMEOUT_MS`, a layer-2 reference-only +vector). ## Performance Contract diff --git a/package.json b/package.json index 422fa7cc0..37651af68 100644 --- a/package.json +++ b/package.json @@ -103,7 +103,9 @@ "build:all": "pnpm build:node && pnpm build:xcuitest", "ad": "node bin/agent-device.mjs", "bench:help-conformance": "node scripts/help-conformance-bench.mjs", - "maestro:conformance": "node --experimental-strip-types scripts/maestro-conformance.ts", + "maestro:conformance": "node --experimental-strip-types --test scripts/maestro-conformance/verify.test.ts scripts/maestro-conformance/differential/run.test.ts scripts/maestro-conformance/differential/invariants.test.ts", + "maestro:conformance:regenerate": "node scripts/maestro-conformance/regenerate.mjs", + "maestro:conformance:differential": "node --experimental-strip-types scripts/maestro-conformance/differential/run.ts", "size": "node scripts/size-report.mjs", "size:markdown": "node scripts/size-report.mjs --json .tmp/size-report.json --markdown .tmp/size-report.md", "perf": "node --experimental-strip-types scripts/perf/run.ts", @@ -131,7 +133,7 @@ "check": "pnpm check:tooling && pnpm check:fallow && pnpm check:unit", "prepack": "pnpm check:mcp-metadata && pnpm build:all && pnpm package:apple-runner:npm && pnpm package:android-snapshot-helper:npm && pnpm package:android-multitouch-helper:npm && pnpm package:android-ime-helper:npm", "typecheck": "tsc -p tsconfig.json", - "test-app:install": "pnpm install --dir examples/test-app --ignore-workspace", + "test-app:install": "pnpm install --dir examples/test-app", "test-app:start": "pnpm --dir examples/test-app start", "test-app:ios": "pnpm --dir examples/test-app ios", "test-app:android": "pnpm --dir examples/test-app android", diff --git a/scripts/maestro-conformance-fixtures/README.md b/scripts/maestro-conformance-fixtures/README.md deleted file mode 100644 index 6ea6bd2fb..000000000 --- a/scripts/maestro-conformance-fixtures/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Maestro Conformance Fixtures - -The harness compares the supported `agent-device` Maestro flow model with a -small, checked-in capture of the Maestro 2.5.1 command model. The upstream tag, -commit, source paths, and SHA-256 values live in -`upstream-maestro-2.5.1.json`. - -The default check is deterministic and does not need Java or Maestro: - -```sh -node --experimental-strip-types scripts/maestro-conformance.ts -``` - -The raw capture is normalized into `normalized-maestro-2.5.1.json`. Regenerate -that file only after reviewing an intentional upstream capture update: - -```sh -node --experimental-strip-types scripts/maestro-conformance.ts --regenerate -``` - -The capture includes fixture source locations so the comparison also protects -`runFlow` include provenance. Timestamps and internal action names are removed -before comparison. diff --git a/scripts/maestro-conformance-fixtures/launch-defaults.yaml b/scripts/maestro-conformance-fixtures/launch-defaults.yaml deleted file mode 100644 index 0bf925311..000000000 --- a/scripts/maestro-conformance-fixtures/launch-defaults.yaml +++ /dev/null @@ -1,3 +0,0 @@ -appId: com.example.pager ---- -- launchApp diff --git a/scripts/maestro-conformance-fixtures/normalized-maestro-2.5.1.json b/scripts/maestro-conformance-fixtures/normalized-maestro-2.5.1.json deleted file mode 100644 index 710900214..000000000 --- a/scripts/maestro-conformance-fixtures/normalized-maestro-2.5.1.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "schemaVersion": 1, - "upstream": { - "project": "mobile-dev-inc/Maestro", - "version": "2.5.1", - "tag": "v2.5.1", - "commit": "a4c7c95f5ba1884858f7e35efa6b8e0165db9448", - "sourceUrl": "https://github.com/mobile-dev-inc/Maestro/tree/v2.5.1", - "artifacts": [ - { - "path": "maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlSwipe.kt", - "role": "YAML swipe classification and percentage endpoint parsing", - "sha256": "1ad0e1c8d80edabc1b5de9390bc64d29117c1d5eff2016faef9f8cf6b9d728e3" - }, - { - "path": "maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlFluentCommand.kt", - "role": "YAML command to Maestro command-model conversion", - "sha256": "6ab05c3932040e20dffa0a683ce0eecb9241e0c9b7bcc0188b697c196797c01d" - }, - { - "path": "maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlLaunchApp.kt", - "role": "launchApp scalar/map decoding", - "sha256": "2e656607e44acb51969afcafe6d75316cf2586f6e0614696f08cab3719f6b615" - }, - { - "path": "maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlRunFlow.kt", - "role": "runFlow file/inline command shape", - "sha256": "a1c50cbd18fd329e9ca91751f324b1caf07aff33616cc4de7ddecaabd3fda66e" - }, - { - "path": "maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlElementSelector.kt", - "role": "selector YAML fields", - "sha256": "907b0c8298d2cdc0a369937f1cc0f1787dfe195a82de6f8b2c46f28d0571446b" - }, - { - "path": "maestro-orchestra-models/src/main/java/maestro/orchestra/Commands.kt", - "role": "LaunchAppCommand, SwipeCommand, and RunFlowCommand model defaults", - "sha256": "64c98d657b649e90e12c0246bfe18465bf2691233f11884ca975f955afbd3836" - }, - { - "path": "maestro-orchestra-models/src/main/java/maestro/orchestra/ElementSelector.kt", - "role": "normalized selector model", - "sha256": "b8e97a99806d992cb02d9edc33f258af7404e33dad9e9ce4786bda86d81171a9" - } - ] - }, - "cases": [ - { - "id": "pager-percentage-swipe", - "flow": "pager-percentage-swipe.yaml", - "expected": [ - { - "kind": "swipe", - "mode": "relative", - "start": [ - 90, - 50 - ], - "end": [ - 10, - 50 - ], - "durationMs": 400, - "source": { - "path": "pager-percentage-swipe.yaml", - "line": 3 - } - } - ] - }, - { - "id": "launch-defaults", - "flow": "launch-defaults.yaml", - "expected": [ - { - "kind": "launchApp", - "appId": "com.example.pager", - "stopApp": true, - "source": { - "path": "launch-defaults.yaml", - "line": 3 - } - } - ] - }, - { - "id": "selectors", - "flow": "selectors.yaml", - "expected": [ - { - "kind": "tapOn", - "selector": { - "text": "Open details" - }, - "source": { - "path": "selectors.yaml", - "line": 3 - } - }, - { - "kind": "tapOn", - "selector": { - "id": "pager-next", - "index": 1, - "childOf": { - "id": "pager" - } - }, - "source": { - "path": "selectors.yaml", - "line": 4 - } - }, - { - "kind": "tapOn", - "selector": { - "text": "Ready", - "enabled": false - }, - "source": { - "path": "selectors.yaml", - "line": 9 - } - } - ] - }, - { - "id": "runflow-include-provenance", - "flow": "runflow-main.yaml", - "expected": [ - { - "kind": "tapOn", - "selector": { - "text": "Before" - }, - "source": { - "path": "runflow-main.yaml", - "line": 3 - } - }, - { - "kind": "launchApp", - "appId": "com.example.include", - "stopApp": true, - "source": { - "path": "runflow-child.yaml", - "line": 3 - } - }, - { - "kind": "tapOn", - "selector": { - "id": "included-button" - }, - "source": { - "path": "runflow-child.yaml", - "line": 4 - } - }, - { - "kind": "tapOn", - "selector": { - "text": "After" - }, - "source": { - "path": "runflow-main.yaml", - "line": 6 - } - } - ] - } - ] -} diff --git a/scripts/maestro-conformance-fixtures/pager-percentage-swipe.yaml b/scripts/maestro-conformance-fixtures/pager-percentage-swipe.yaml deleted file mode 100644 index d5f89d32e..000000000 --- a/scripts/maestro-conformance-fixtures/pager-percentage-swipe.yaml +++ /dev/null @@ -1,7 +0,0 @@ -appId: com.example.pager ---- -- swipe: - from: - id: pager-view - start: 90%, 50% - end: 10%, 50% diff --git a/scripts/maestro-conformance-fixtures/runflow-main.yaml b/scripts/maestro-conformance-fixtures/runflow-main.yaml deleted file mode 100644 index 1a57ec86c..000000000 --- a/scripts/maestro-conformance-fixtures/runflow-main.yaml +++ /dev/null @@ -1,6 +0,0 @@ -appId: com.example.include ---- -- tapOn: Before -- runFlow: - file: runflow-child.yaml -- tapOn: After diff --git a/scripts/maestro-conformance-fixtures/selectors.yaml b/scripts/maestro-conformance-fixtures/selectors.yaml deleted file mode 100644 index ad647cf7c..000000000 --- a/scripts/maestro-conformance-fixtures/selectors.yaml +++ /dev/null @@ -1,11 +0,0 @@ -appId: com.example.pager ---- -- tapOn: Open details -- tapOn: - id: pager-next - index: 1 - childOf: - id: pager -- tapOn: - text: Ready - enabled: false diff --git a/scripts/maestro-conformance-fixtures/upstream-maestro-2.5.1.json b/scripts/maestro-conformance-fixtures/upstream-maestro-2.5.1.json deleted file mode 100644 index 19df4dbbc..000000000 --- a/scripts/maestro-conformance-fixtures/upstream-maestro-2.5.1.json +++ /dev/null @@ -1,174 +0,0 @@ -{ - "schemaVersion": 1, - "upstream": { - "project": "mobile-dev-inc/Maestro", - "version": "2.5.1", - "tag": "v2.5.1", - "commit": "a4c7c95f5ba1884858f7e35efa6b8e0165db9448", - "sourceUrl": "https://github.com/mobile-dev-inc/Maestro/tree/v2.5.1", - "artifacts": [ - { - "path": "maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlSwipe.kt", - "role": "YAML swipe classification and percentage endpoint parsing", - "sha256": "1ad0e1c8d80edabc1b5de9390bc64d29117c1d5eff2016faef9f8cf6b9d728e3" - }, - { - "path": "maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlFluentCommand.kt", - "role": "YAML command to Maestro command-model conversion", - "sha256": "6ab05c3932040e20dffa0a683ce0eecb9241e0c9b7bcc0188b697c196797c01d" - }, - { - "path": "maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlLaunchApp.kt", - "role": "launchApp scalar/map decoding", - "sha256": "2e656607e44acb51969afcafe6d75316cf2586f6e0614696f08cab3719f6b615" - }, - { - "path": "maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlRunFlow.kt", - "role": "runFlow file/inline command shape", - "sha256": "a1c50cbd18fd329e9ca91751f324b1caf07aff33616cc4de7ddecaabd3fda66e" - }, - { - "path": "maestro-orchestra/src/main/java/maestro/orchestra/yaml/YamlElementSelector.kt", - "role": "selector YAML fields", - "sha256": "907b0c8298d2cdc0a369937f1cc0f1787dfe195a82de6f8b2c46f28d0571446b" - }, - { - "path": "maestro-orchestra-models/src/main/java/maestro/orchestra/Commands.kt", - "role": "LaunchAppCommand, SwipeCommand, and RunFlowCommand model defaults", - "sha256": "64c98d657b649e90e12c0246bfe18465bf2691233f11884ca975f955afbd3836" - }, - { - "path": "maestro-orchestra-models/src/main/java/maestro/orchestra/ElementSelector.kt", - "role": "normalized selector model", - "sha256": "b8e97a99806d992cb02d9edc33f258af7404e33dad9e9ce4786bda86d81171a9" - } - ] - }, - "cases": [ - { - "id": "pager-percentage-swipe", - "flow": "pager-percentage-swipe.yaml", - "commands": [ - { - "type": "SwipeCommand", - "startRelative": "90%, 50%", - "endRelative": "10%, 50%", - "duration": 400, - "source": { - "path": "pager-percentage-swipe.yaml", - "line": 3 - } - } - ] - }, - { - "id": "launch-defaults", - "flow": "launch-defaults.yaml", - "commands": [ - { - "type": "LaunchAppCommand", - "appId": "com.example.pager", - "source": { - "path": "launch-defaults.yaml", - "line": 3 - } - } - ] - }, - { - "id": "selectors", - "flow": "selectors.yaml", - "commands": [ - { - "type": "TapOnElementCommand", - "selector": { - "textRegex": "Open details" - }, - "source": { - "path": "selectors.yaml", - "line": 3 - } - }, - { - "type": "TapOnElementCommand", - "selector": { - "idRegex": "pager-next", - "index": 1, - "childOf": { - "idRegex": "pager" - } - }, - "source": { - "path": "selectors.yaml", - "line": 4 - } - }, - { - "type": "TapOnElementCommand", - "selector": { - "textRegex": "Ready", - "enabled": false - }, - "source": { - "path": "selectors.yaml", - "line": 9 - } - } - ] - }, - { - "id": "runflow-include-provenance", - "flow": "runflow-main.yaml", - "commands": [ - { - "type": "TapOnElementCommand", - "selector": { - "textRegex": "Before" - }, - "source": { - "path": "runflow-main.yaml", - "line": 3 - } - }, - { - "type": "RunFlowCommand", - "sourceDescription": "runflow-child.yaml", - "source": { - "path": "runflow-main.yaml", - "line": 4 - }, - "commands": [ - { - "type": "LaunchAppCommand", - "appId": "com.example.include", - "source": { - "path": "runflow-child.yaml", - "line": 3 - } - }, - { - "type": "TapOnElementCommand", - "selector": { - "idRegex": "included-button" - }, - "source": { - "path": "runflow-child.yaml", - "line": 4 - } - } - ] - }, - { - "type": "TapOnElementCommand", - "selector": { - "textRegex": "After" - }, - "source": { - "path": "runflow-main.yaml", - "line": 6 - } - } - ] - } - ] -} diff --git a/scripts/maestro-conformance-model.ts b/scripts/maestro-conformance-model.ts deleted file mode 100644 index 225ee2785..000000000 --- a/scripts/maestro-conformance-model.ts +++ /dev/null @@ -1,342 +0,0 @@ -import fs from 'node:fs'; -import path from 'node:path'; -import { parseMaestroProgram } from '../src/compat/maestro/program-ir-parser.ts'; -import { MAESTRO_COMPATIBILITY_PRESETS } from '../src/compat/maestro/compatibility-policy.ts'; -import type { - MaestroCommand, - MaestroProgram, - MaestroSelector, -} from '../src/compat/maestro/program-ir.ts'; -import { normalizeUpstreamSelector } from './maestro-conformance-selectors.ts'; -import type { - NormalizedAction, - NormalizedCase, - NormalizedFixture, - NormalizedSelector, - NormalizedSource, - RawCase, - RawCommand, - RawFixture, - UpstreamSource, -} from './maestro-conformance-types.ts'; -import { readOptionalString, readRequiredRecord } from './maestro-conformance-values.ts'; -import { stripUndefined } from '../src/utils/parsing.ts'; - -const UPSTREAM_DEFAULT_SWIPE_DURATION_MS = 400; - -export function normalizeUpstreamFixture( - fixture: RawFixture, - fixtureDirectory: string, -): NormalizedFixture { - return { - schemaVersion: 1, - upstream: fixture.upstream, - cases: fixture.cases.map((entry) => ({ - id: entry.id, - flow: entry.flow, - expected: entry.commands.flatMap((command) => - normalizeUpstreamCommand(command, fixtureDirectory, { - path: entry.flow, - line: 1, - }), - ), - })), - }; -} - -export function normalizeAgentCase(fixture: RawCase, fixtureDirectory: string): NormalizedCase { - const flowPath = resolveFixturePath(fixtureDirectory, fixture.flow); - const program = parseMaestroProgram(fs.readFileSync(flowPath, 'utf8'), { sourcePath: flowPath }); - return { - id: fixture.id, - flow: fixture.flow, - expected: normalizeAgentProgram(program, fixtureDirectory), - }; -} - -function normalizeAgentProgram( - program: MaestroProgram, - fixtureDirectory: string, -): NormalizedAction[] { - return [ - ...(program.config.onFlowStart ?? []), - ...program.commands, - ...(program.config.onFlowComplete ?? []), - ].flatMap((command) => normalizeAgentCommand(command, program, fixtureDirectory)); -} - -function normalizeAgentCommand( - command: MaestroCommand, - program: MaestroProgram, - fixtureDirectory: string, -): NormalizedAction[] { - const source = normalizeSource(command.source, fixtureDirectory); - switch (command.kind) { - case 'runFlow': - return normalizeAgentRunFlow(command, program, fixtureDirectory); - case 'launchApp': { - const appId = command.appId ?? program.config.appId; - if (!appId) throw new Error('launchApp conformance fixture requires appId.'); - return [{ kind: 'launchApp', appId, stopApp: command.stopApp !== false, source }]; - } - case 'swipe': - return [normalizeAgentSwipe(command.gesture, source)]; - case 'tapOn': - return [normalizeAgentTap(command, source)]; - case 'assertVisible': - case 'assertNotVisible': - return [ - { - kind: command.kind, - selector: normalizeTypedSelector(command.target), - timeoutMs: MAESTRO_COMPATIBILITY_PRESETS.command.targetLookupTimeoutMs, - source, - }, - ]; - default: - throw new Error(`Unsupported typed command in conformance fixture: ${command.kind}`); - } -} - -function normalizeAgentRunFlow( - command: Extract, - program: MaestroProgram, - fixtureDirectory: string, -): NormalizedAction[] { - if (command.include.kind === 'commands') { - return command.include.commands.flatMap((nested) => - normalizeAgentCommand(nested, program, fixtureDirectory), - ); - } - const parentPath = command.source.path ?? program.source.path; - if (!parentPath) throw new Error('File runFlow requires source path provenance.'); - const includePath = path.resolve(path.dirname(parentPath), command.include.path); - const included = parseMaestroProgram(fs.readFileSync(includePath, 'utf8'), { - sourcePath: includePath, - }); - return normalizeAgentProgram(included, fixtureDirectory); -} - -function normalizeAgentTap( - command: Extract, - source: NormalizedSource, -): NormalizedAction { - if (command.target.space !== 'target') { - throw new Error('tapOn conformance fixtures require selector targets.'); - } - return { - kind: 'tapOn', - selector: stripUndefined({ - ...normalizeTypedSelector(command.target.selector), - index: command.index, - childOf: command.childOf ? normalizeTypedSelector(command.childOf) : undefined, - }), - source, - }; -} - -function normalizeAgentSwipe( - gesture: Extract['gesture'], - source: NormalizedSource, -): NormalizedAction { - const durationMs = gesture.duration ?? MAESTRO_COMPATIBILITY_PRESETS.command.swipeDurationMs; - if (gesture.kind === 'screen') { - return { kind: 'swipe', mode: 'direction', direction: gesture.direction, durationMs, source }; - } - if (gesture.kind === 'target') { - throw new Error('Target-relative swipe is not part of the conformance fixture set.'); - } - return { - kind: 'swipe', - mode: gesture.start.space === 'percent' ? 'relative' : 'absolute', - start: [gesture.start.x, gesture.start.y], - end: [gesture.end.x, gesture.end.y], - durationMs, - source, - }; -} - -function normalizeTypedSelector(selector: MaestroSelector): NormalizedSelector { - const text = selector.text ?? selector.label; - return stripUndefined({ id: selector.id, text, enabled: selector.enabled, selected: selector.selected }); -} - -function normalizeUpstreamCommand( - command: RawCommand, - fixtureDirectory: string, - fallbackSource: UpstreamSource, -): NormalizedAction[] { - const source = normalizeSource(command.source ?? fallbackSource, fixtureDirectory); - - switch (command.type) { - case 'RunFlowCommand': { - if (!command.commands) throw new Error('RunFlowCommand artifact is missing commands.'); - return command.commands.flatMap((nested) => - normalizeUpstreamCommand(nested, fixtureDirectory, source), - ); - } - case 'LaunchAppCommand': - return [ - { - kind: 'launchApp', - appId: requiredString(command, 'appId'), - stopApp: command.stopApp !== false, - source, - }, - ]; - case 'SwipeCommand': - return [normalizeUpstreamSwipe(command, source)]; - case 'TapOnElementCommand': - return [ - { - kind: 'tapOn', - selector: normalizeUpstreamSelector(requiredRecord(command, 'selector')), - source, - }, - ]; - case 'AssertConditionCommand': - return [normalizeUpstreamAssertion(command, source)]; - default: - throw new Error(`Unsupported upstream command artifact: ${command.type}`); - } -} - -function normalizeUpstreamSwipe(command: RawCommand, source: NormalizedSource): NormalizedAction { - const durationMs = integerOrDefault(command.duration, UPSTREAM_DEFAULT_SWIPE_DURATION_MS); - const startRelative = readOptionalString(command, 'startRelative'); - const endRelative = readOptionalString(command, 'endRelative'); - if (startRelative !== undefined || endRelative !== undefined) { - if (startRelative === undefined || endRelative === undefined) { - throw new Error('SwipeCommand artifact must include both relative endpoints.'); - } - return { - kind: 'swipe', - mode: 'relative', - start: parsePoint(startRelative, '%'), - end: parsePoint(endRelative, '%'), - durationMs, - source, - }; - } - - const direction = readOptionalString(command, 'direction'); - if (direction !== undefined) { - return { - kind: 'swipe', - mode: 'direction', - direction: direction.toLowerCase(), - durationMs, - source, - }; - } - - const startPoint = optionalPoint(command, 'startPoint'); - const endPoint = optionalPoint(command, 'endPoint'); - if (startPoint && endPoint) { - return { - kind: 'swipe', - mode: 'absolute', - start: startPoint, - end: endPoint, - durationMs, - source, - }; - } - throw new Error('SwipeCommand artifact has no supported gesture shape.'); -} - -function normalizeUpstreamAssertion( - command: RawCommand, - source: NormalizedSource, -): NormalizedAction { - const condition = requiredRecord(command, 'condition'); - const visible = condition.visible; - const notVisible = condition.notVisible; - if (visible !== undefined && notVisible === undefined) { - return { - kind: 'assertVisible', - selector: normalizeUpstreamSelector(requiredRecordValue(visible, 'condition.visible')), - timeoutMs: integerOrDefault(command.timeout, 17000), - source, - }; - } - if (notVisible !== undefined && visible === undefined) { - return { - kind: 'assertNotVisible', - selector: normalizeUpstreamSelector(requiredRecordValue(notVisible, 'condition.notVisible')), - timeoutMs: integerOrDefault(command.timeout, 17000), - source, - }; - } - throw new Error('AssertConditionCommand artifact must contain one condition.'); -} - -function optionalPoint(record: Record, key: string): [number, number] | undefined { - const value = record[key]; - if (value === undefined) return undefined; - if (Array.isArray(value) && value.length === 2) { - return [numberValue(value[0], `${key}[0]`), numberValue(value[1], `${key}[1]`)] as [ - number, - number, - ]; - } - if (typeof value === 'string') return parsePoint(value, ''); - throw new Error(`Unsupported ${key} point artifact.`); -} - -function parsePoint(value: string, suffix: string): [number, number] { - const escapedSuffix = suffix === '%' ? '%' : ''; - const match = value.match( - new RegExp( - `^\\s*(\\d+(?:\\.\\d+)?)${escapedSuffix}\\s*,\\s*(\\d+(?:\\.\\d+)?)${escapedSuffix}\\s*$`, - ), - ); - if (!match) throw new Error(`Invalid ${suffix ? 'relative ' : ''}point: ${value}`); - return [Number(match[1]), Number(match[2])]; -} - -function integerOrDefault(value: unknown, fallback: number): number { - if (value === undefined || value === null || value === '') return fallback; - const number = typeof value === 'number' ? value : Number(value); - if (!Number.isInteger(number) || number < 0) - throw new Error(`Expected non-negative integer, got ${value}`); - return number; -} - -function numberValue(value: unknown, name: string): number { - const number = typeof value === 'number' ? value : Number(value); - if (!Number.isFinite(number)) throw new Error(`Invalid ${name}: ${String(value)}`); - return number; -} - -function requiredString(record: Record, key: string): string { - const value = readOptionalString(record, key); - if (value === undefined || value.length === 0) throw new Error(`${key} is required.`); - return value; -} - -function requiredRecord(record: Record, key: string): Record { - return requiredRecordValue(record[key], key); -} - -function requiredRecordValue(value: unknown, name: string): Record { - return readRequiredRecord(value, name); -} - -function resolveFixturePath(fixtureDirectory: string, relativePath: string): string { - const resolved = path.resolve(fixtureDirectory, relativePath); - const relative = path.relative(fixtureDirectory, resolved); - if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) { - throw new Error(`Fixture path escapes fixture directory: ${relativePath}`); - } - return resolved; -} - -function normalizeSource(source: UpstreamSource, fixtureDirectory: string): NormalizedSource { - const resolved = path.resolve(fixtureDirectory, source.path); - const relative = path.relative(fixtureDirectory, resolved); - if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) { - throw new Error(`Source path escapes fixture directory: ${source.path}`); - } - return { path: relative.split(path.sep).join('/'), line: source.line }; -} diff --git a/scripts/maestro-conformance-selectors.ts b/scripts/maestro-conformance-selectors.ts deleted file mode 100644 index f373d62ff..000000000 --- a/scripts/maestro-conformance-selectors.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { NormalizedSelector } from './maestro-conformance-types.ts'; -import { readOptionalString, readRequiredRecord } from './maestro-conformance-values.ts'; - -export function normalizeUpstreamSelector(selector: Record): NormalizedSelector { - const result: NormalizedSelector = {}; - const text = readOptionalString(selector, 'textRegex') ?? readOptionalString(selector, 'text'); - const id = readOptionalString(selector, 'idRegex') ?? readOptionalString(selector, 'id'); - if (text !== undefined) result.text = text; - if (id !== undefined) result.id = id; - if (selector.index !== undefined) result.index = numberValue(selector.index, 'selector index'); - if (selector.childOf !== undefined) { - result.childOf = normalizeUpstreamSelector( - readRequiredRecord(selector.childOf, 'selector.childOf'), - ); - } - readSelectorState(selector, result, 'enabled'); - readSelectorState(selector, result, 'selected'); - if (Object.keys(result).length === 0) throw new Error('Selector artifact is empty.'); - return result; -} - -function readSelectorState( - record: Record, - result: NormalizedSelector, - key: 'enabled' | 'selected', -): void { - const value = record[key]; - if (value === undefined) return; - if (typeof value !== 'boolean') throw new Error(`Selector ${key} must be boolean.`); - result[key] = value; -} - -function numberValue(value: unknown, name: string): number { - const number = typeof value === 'number' ? value : Number(value); - if (!Number.isInteger(number) || number < 0) - throw new Error(`${name} must be a non-negative integer.`); - return number; -} diff --git a/scripts/maestro-conformance-types.ts b/scripts/maestro-conformance-types.ts deleted file mode 100644 index e99e0e721..000000000 --- a/scripts/maestro-conformance-types.ts +++ /dev/null @@ -1,84 +0,0 @@ -export type UpstreamArtifact = { - path: string; - role: string; - sha256: string; -}; - -export type UpstreamPin = { - project: string; - version: string; - tag: string; - commit: string; - sourceUrl: string; - artifacts: UpstreamArtifact[]; -}; - -export type UpstreamSource = { - path: string; - line: number; -}; - -export type RawCommand = { - type: string; - source?: UpstreamSource; - commands?: RawCommand[]; - [key: string]: unknown; -}; - -export type RawCase = { - id: string; - flow: string; - commands: RawCommand[]; -}; - -export type RawFixture = { - schemaVersion: 1; - upstream: UpstreamPin; - cases: RawCase[]; -}; - -export type NormalizedSource = UpstreamSource; - -export type NormalizedSelector = { - id?: string; - text?: string; - index?: number; - childOf?: NormalizedSelector; - enabled?: boolean; - selected?: boolean; -}; - -export type NormalizedAction = - | { - kind: 'launchApp'; - appId: string; - stopApp: boolean; - source: NormalizedSource; - } - | { - kind: 'swipe'; - mode: 'relative' | 'absolute' | 'direction'; - start?: [number, number]; - end?: [number, number]; - direction?: string; - durationMs: number; - source: NormalizedSource; - } - | { - kind: 'tapOn' | 'assertVisible' | 'assertNotVisible'; - selector: NormalizedSelector; - timeoutMs?: number; - source: NormalizedSource; - }; - -export type NormalizedCase = { - id: string; - flow: string; - expected: NormalizedAction[]; -}; - -export type NormalizedFixture = { - schemaVersion: 1; - upstream: UpstreamPin; - cases: NormalizedCase[]; -}; diff --git a/scripts/maestro-conformance-values.ts b/scripts/maestro-conformance-values.ts deleted file mode 100644 index 8e68b1555..000000000 --- a/scripts/maestro-conformance-values.ts +++ /dev/null @@ -1,16 +0,0 @@ -export function readRequiredRecord(value: unknown, name: string): Record { - if (typeof value !== 'object' || value === null || Array.isArray(value)) { - throw new Error(`${name} must be an object.`); - } - return value as Record; -} - -export function readOptionalString( - record: Record, - key: string, -): string | undefined { - const value = record[key]; - if (value === undefined || value === null) return undefined; - if (typeof value !== 'string') throw new Error(`${key} must be a string.`); - return value; -} diff --git a/scripts/maestro-conformance.test.ts b/scripts/maestro-conformance.test.ts deleted file mode 100644 index 06a132161..000000000 --- a/scripts/maestro-conformance.test.ts +++ /dev/null @@ -1,91 +0,0 @@ -import assert from 'node:assert/strict'; -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import { test } from 'vitest'; -import { - MAESTRO_CONFORMANCE_FIXTURE_DIRECTORY, - MAESTRO_CONFORMANCE_NORMALIZED_FIXTURE, - checkConformance, - regenerateConformance, -} from './maestro-conformance.ts'; - -test('checked-in Maestro 2.5.1 fixtures match the agent-device model', () => { - const result = checkConformance(); - - assert.equal(result.upstream.version, '2.5.1'); - assert.equal(result.upstream.commit, 'a4c7c95f5ba1884858f7e35efa6b8e0165db9448'); - assert.deepEqual( - result.cases.map((entry) => entry.id), - ['pager-percentage-swipe', 'launch-defaults', 'selectors', 'runflow-include-provenance'], - ); -}); - -test('normalization covers percentage swipes, launch defaults, selectors, and includes', () => { - const result = checkConformance(); - const pager = result.cases.find((entry) => entry.id === 'pager-percentage-swipe'); - const launch = result.cases.find((entry) => entry.id === 'launch-defaults'); - const selectors = result.cases.find((entry) => entry.id === 'selectors'); - const include = result.cases.find((entry) => entry.id === 'runflow-include-provenance'); - - assert.deepEqual(pager?.expected[0], { - kind: 'swipe', - mode: 'relative', - start: [90, 50], - end: [10, 50], - durationMs: 400, - source: { path: 'pager-percentage-swipe.yaml', line: 3 }, - }); - assert.deepEqual(launch?.expected[0], { - kind: 'launchApp', - appId: 'com.example.pager', - stopApp: true, - source: { path: 'launch-defaults.yaml', line: 3 }, - }); - assert.deepEqual( - selectors?.expected.map((entry) => entry), - [ - { - kind: 'tapOn', - selector: { text: 'Open details' }, - source: { path: 'selectors.yaml', line: 3 }, - }, - { - kind: 'tapOn', - selector: { id: 'pager-next', index: 1, childOf: { id: 'pager' } }, - source: { path: 'selectors.yaml', line: 4 }, - }, - { - kind: 'tapOn', - selector: { text: 'Ready', enabled: false }, - source: { path: 'selectors.yaml', line: 9 }, - }, - ], - ); - assert.deepEqual( - include?.expected.map((entry) => entry.source), - [ - { path: 'runflow-main.yaml', line: 3 }, - { path: 'runflow-child.yaml', line: 3 }, - { path: 'runflow-child.yaml', line: 4 }, - { path: 'runflow-main.yaml', line: 6 }, - ], - ); -}); - -test('regeneration is explicit and produces a checkable normalized fixture', () => { - const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'maestro-conformance-')); - try { - fs.cpSync(MAESTRO_CONFORMANCE_FIXTURE_DIRECTORY, temporaryDirectory, { recursive: true }); - fs.writeFileSync( - path.join(temporaryDirectory, MAESTRO_CONFORMANCE_NORMALIZED_FIXTURE), - '{"schemaVersion":1,"cases":[]}\n', - ); - - const regenerated = regenerateConformance({ fixtureDirectory: temporaryDirectory }); - assert.equal(regenerated.cases.length, 4); - assert.doesNotThrow(() => checkConformance({ fixtureDirectory: temporaryDirectory })); - } finally { - fs.rmSync(temporaryDirectory, { recursive: true, force: true }); - } -}); diff --git a/scripts/maestro-conformance.ts b/scripts/maestro-conformance.ts deleted file mode 100644 index 456da096c..000000000 --- a/scripts/maestro-conformance.ts +++ /dev/null @@ -1,255 +0,0 @@ -import fs from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { normalizeAgentCase, normalizeUpstreamFixture } from './maestro-conformance-model.ts'; -import type { - NormalizedCase, - NormalizedFixture, - RawCase, - RawCommand, - RawFixture, - UpstreamArtifact, - UpstreamPin, - UpstreamSource, -} from './maestro-conformance-types.ts'; -import { readRequiredRecord } from './maestro-conformance-values.ts'; - -export const MAESTRO_CONFORMANCE_FIXTURE_DIRECTORY = path.resolve( - path.dirname(fileURLToPath(import.meta.url)), - 'maestro-conformance-fixtures', -); -export const MAESTRO_CONFORMANCE_RAW_FIXTURE = 'upstream-maestro-2.5.1.json'; -export const MAESTRO_CONFORMANCE_NORMALIZED_FIXTURE = 'normalized-maestro-2.5.1.json'; -const PINNED_UPSTREAM = { - project: 'mobile-dev-inc/Maestro', - version: '2.5.1', - tag: 'v2.5.1', - commit: 'a4c7c95f5ba1884858f7e35efa6b8e0165db9448', -} as const; - -export type ConformanceCheckResult = { - upstream: UpstreamPin; - cases: NormalizedCase[]; -}; - -export function checkConformance( - options: { - fixtureDirectory?: string; - } = {}, -): ConformanceCheckResult { - const fixtureDirectory = options.fixtureDirectory ?? MAESTRO_CONFORMANCE_FIXTURE_DIRECTORY; - const raw = readRawFixture(path.join(fixtureDirectory, MAESTRO_CONFORMANCE_RAW_FIXTURE)); - const normalized = readNormalizedFixture( - path.join(fixtureDirectory, MAESTRO_CONFORMANCE_NORMALIZED_FIXTURE), - ); - assertPinnedUpstream(raw.upstream); - compareJson(normalized.upstream, raw.upstream, 'normalized fixture upstream pin'); - - const regenerated = normalizeUpstreamFixture(raw, fixtureDirectory); - compareJson(regenerated.cases, normalized.cases, 'normalized upstream cases'); - - const actual = raw.cases.map((entry) => normalizeAgentCase(entry, fixtureDirectory)); - compareJson(actual, normalized.cases, 'agent-device flow model'); - return { upstream: raw.upstream, cases: actual }; -} - -export function regenerateConformance( - options: { - fixtureDirectory?: string; - } = {}, -): NormalizedFixture { - const fixtureDirectory = options.fixtureDirectory ?? MAESTRO_CONFORMANCE_FIXTURE_DIRECTORY; - const raw = readRawFixture(path.join(fixtureDirectory, MAESTRO_CONFORMANCE_RAW_FIXTURE)); - assertPinnedUpstream(raw.upstream); - const normalized = normalizeUpstreamFixture(raw, fixtureDirectory); - fs.writeFileSync( - path.join(fixtureDirectory, MAESTRO_CONFORMANCE_NORMALIZED_FIXTURE), - `${JSON.stringify(normalized, null, 2)}\n`, - ); - return normalized; -} - -function readRawFixture(filePath: string): RawFixture { - const value = readJson(filePath); - const record = requiredRecord(value, filePath); - if (record.schemaVersion !== 1) throw new Error(`${filePath} has an unsupported schemaVersion.`); - const cases = requiredArray(record.cases, `${filePath}.cases`).map((entry, index) => - readRawCase(entry, `${filePath}.cases[${index}]`), - ); - return { - schemaVersion: 1, - upstream: readUpstreamPin(record.upstream, `${filePath}.upstream`), - cases, - }; -} - -function readNormalizedFixture(filePath: string): NormalizedFixture { - const value = readJson(filePath); - const record = requiredRecord(value, filePath); - if (record.schemaVersion !== 1) throw new Error(`${filePath} has an unsupported schemaVersion.`); - return { - schemaVersion: 1, - upstream: readUpstreamPin(record.upstream, `${filePath}.upstream`), - cases: requiredArray(record.cases, `${filePath}.cases`).map((entry, index) => { - const caseRecord = requiredRecord(entry, `${filePath}.cases[${index}]`); - return { - id: requiredString(caseRecord.id, `${filePath}.cases[${index}].id`), - flow: requiredString(caseRecord.flow, `${filePath}.cases[${index}].flow`), - expected: requiredArray( - caseRecord.expected, - `${filePath}.cases[${index}].expected`, - ) as NormalizedCase['expected'], - }; - }), - }; -} - -function readRawCase(value: unknown, name: string): RawCase { - const record = requiredRecord(value, name); - return { - id: requiredString(record.id, `${name}.id`), - flow: requiredString(record.flow, `${name}.flow`), - commands: requiredArray(record.commands, `${name}.commands`).map((entry, index) => - readRawCommand(entry, `${name}.commands[${index}]`), - ), - }; -} - -function readRawCommand(value: unknown, name: string): RawCommand { - const record = requiredRecord(value, name); - const command: RawCommand = { - ...record, - type: requiredString(record.type, `${name}.type`), - }; - if (record.source !== undefined) command.source = readSource(record.source, `${name}.source`); - if (record.commands !== undefined) { - command.commands = requiredArray(record.commands, `${name}.commands`).map((entry, index) => - readRawCommand(entry, `${name}.commands[${index}]`), - ); - } - return command; -} - -function readUpstreamPin(value: unknown, name: string): UpstreamPin { - const record = requiredRecord(value, name); - const artifacts = requiredArray(record.artifacts, `${name}.artifacts`).map((entry, index) => { - const artifact = requiredRecord(entry, `${name}.artifacts[${index}]`); - return { - path: requiredString(artifact.path, `${name}.artifacts[${index}].path`), - role: requiredString(artifact.role, `${name}.artifacts[${index}].role`), - sha256: requiredString(artifact.sha256, `${name}.artifacts[${index}].sha256`), - } satisfies UpstreamArtifact; - }); - return { - project: requiredString(record.project, `${name}.project`), - version: requiredString(record.version, `${name}.version`), - tag: requiredString(record.tag, `${name}.tag`), - commit: requiredString(record.commit, `${name}.commit`), - sourceUrl: requiredString(record.sourceUrl, `${name}.sourceUrl`), - artifacts, - }; -} - -function readSource(value: unknown, name: string): UpstreamSource { - const record = requiredRecord(value, name); - const line = record.line; - if (typeof line !== 'number' || !Number.isInteger(line) || line < 1) { - throw new Error(`${name}.line must be a positive integer.`); - } - return { - path: requiredString(record.path, `${name}.path`), - line, - }; -} - -function readJson(filePath: string): unknown { - return JSON.parse(fs.readFileSync(filePath, 'utf8')) as unknown; -} - -function requiredRecord(value: unknown, name: string): Record { - return readRequiredRecord(value, name); -} - -function requiredArray(value: unknown, name: string): unknown[] { - if (!Array.isArray(value)) throw new Error(`${name} must be an array.`); - return value; -} - -function requiredString(value: unknown, name: string): string { - if (typeof value !== 'string' || value.length === 0) throw new Error(`${name} must be a string.`); - return value; -} - -function assertPinnedUpstream(upstream: UpstreamPin): void { - for (const key of ['project', 'version', 'tag', 'commit'] as const) { - if (upstream[key] !== PINNED_UPSTREAM[key]) { - throw new Error( - `Upstream fixture must pin ${key}=${PINNED_UPSTREAM[key]}; found ${upstream[key]}.`, - ); - } - } -} - -function compareJson(actual: unknown, expected: unknown, name: string): void { - const actualJson = JSON.stringify(actual, null, 2); - const expectedJson = JSON.stringify(expected, null, 2); - if (actualJson === expectedJson) return; - throw new Error( - `${name} mismatch. Run node --experimental-strip-types scripts/maestro-conformance.ts --regenerate only after reviewing the upstream capture.\nExpected:\n${expectedJson}\nActual:\n${actualJson}`, - ); -} - -function parseMode(args: readonly string[]): 'check' | 'regenerate' | 'help' { - let mode: 'check' | 'regenerate' | 'help' = 'check'; - for (const arg of args) { - if (arg === '--help' || arg === '-h') { - mode = 'help'; - continue; - } - if (arg === '--check' || arg === '--regenerate') { - const next = arg === '--check' ? 'check' : 'regenerate'; - if (mode !== 'check' && mode !== next) throw new Error('Choose only one conformance mode.'); - mode = next; - continue; - } - throw new Error(`Unknown argument: ${arg}`); - } - return mode; -} - -function printHelp(): void { - console.log( - 'Usage: node --experimental-strip-types scripts/maestro-conformance.ts [--check|--regenerate]', - ); - console.log(''); - console.log( - ' --check Compare the parser against checked-in normalized fixtures (default).', - ); - console.log(' --regenerate Rebuild the normalized fixture from the pinned upstream capture.'); -} - -function main(args: readonly string[]): void { - try { - const mode = parseMode(args); - if (mode === 'help') { - printHelp(); - return; - } - if (mode === 'regenerate') { - regenerateConformance(); - console.log(`Regenerated ${MAESTRO_CONFORMANCE_NORMALIZED_FIXTURE}.`); - return; - } - const result = checkConformance(); - console.log( - `Maestro ${result.upstream.version} conformance: ${result.cases.length} cases passed.`, - ); - } catch (error) { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; - } -} - -if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { - main(process.argv.slice(2)); -} diff --git a/scripts/maestro-conformance/README.md b/scripts/maestro-conformance/README.md new file mode 100644 index 000000000..bc1bb2a24 --- /dev/null +++ b/scripts/maestro-conformance/README.md @@ -0,0 +1,164 @@ +# Maestro conformance oracle + +A three-layer oracle that proves the agent-device Maestro engine (`src/compat/maestro`) +stays faithful to a version-pinned upstream Maestro. It replaces the original +hand-typed parser fixture, whose transcribed expectations let four bug classes +slip through during #1217. Every expected value here is **generated from the +pinned upstream artifacts** — hand transcription is the failure mode this +replaces (issue #1274). + +Pinned upstream: `dev.mobile:*:2.5.1` (`v2.5.1` / `a4c7c95f`), see +[`pinned-upstream.json`](./pinned-upstream.json). + +## Layers + +| Layer | What it proves | Generated by | Runs in | +|-------|----------------|--------------|---------| +| 1 — parser | Our parser accepts/rejects/normalizes each flow like upstream | `maestro-orchestra`'s `YamlCommandReader` over the corpus | `node --test`, per-PR | +| 2 — semantics | Our geometry/retry/timing constants match upstream | ASM-read bytecode constants + parser-observed model defaults | `node --test`, per-PR | +| 3 — differential | Outcome parity on a device, plus engine-side timing invariants | Real Maestro vs `agent-device test` | dispatch-only (see below) | + +Layers 1–2 are checked-in generated fixtures (`fixtures/`) verified +deterministically with **no Java**. Layer 3 needs a device and the `maestro` CLI. + +### How "generated from upstream" is enforced + +Per-PR CI cannot re-derive the fixtures — that needs Java, and staying Java-free +on the normal path is a design constraint. So enforcement is two-layered: + +1. **Per-PR:** every fixture carries a `contentHash` seal over its generated + content, recomputed on each verify run. Editing a captured command or constant + by hand breaks the seal. This is tamper-**evident** — it makes casual or + accidental hand-editing impossible, but a determined editor could recompute it. +2. **Scheduled (`conformance-regenerate`):** actually re-runs the JVM harness + against the pinned jars and fails if the checked-in fixtures differ by a byte. + Forgery cannot survive a real re-derivation. + +Without (2), "generated from upstream" would be documentation. Do not weaken +either half: together they are what stops this oracle from decaying back into the +hand-typed fixture it replaced. + +### What layer 3 actually proves + +Read `scenarios.ts` literally. Cross-engine comparison is **outcome parity** — it +only catches a divergence severe enough to fail the flow. It cannot see settle +latching, retap counts, or a 1px truncation difference. Where finer behavior +matters we assert it **engine-side** via `engineInvariants` over agent-device's +own `replay-timing.ndjson`. + +Layer-3 flows live in `differential/flows/` and drive the **real fixture app** +(`examples/test-app`, `com.callstack.agentdevicelab`), which the workflow builds +and installs. They are deliberately **not** the layer-1 corpus: those flows exist +only to be *parsed* — they name a fictional `com.example.app` and elements that +exist on no device — so a device run against them would fail before exercising +any runtime behavior, making the settle detector silently vacuous. A test +enforces the separation, and the workflow hard-fails if the app is not installed. + +### Declared divergences (`knownDivergence`) + +Layer 3 has the same contract as layer 1: **every divergence is a decision on the +record.** When the differential catches a real engine bug, the instrument does not +block on repairing what it just measured — the scenario declares it with a +`knownDivergence: { reason, tracking }`, the scheduled run stays green on that +known gap, and only *undeclared* divergences fail. + +Two rules keep that from rotting, both enforced by `run.test.ts` / the runner +rather than by good intentions: + +- **`tracking` is required** and must be a real issue URL. A declared divergence + with nothing behind it is how "temporarily expected" silently becomes permanent. +- **A stale declaration fails.** If a declared-divergent scenario starts passing, + the run goes red until the declaration is removed — so the fix PR must delete + it, and the oracle then enforces that the gap stays closed. The differential is + the acceptance test for its own findings. + +A green run still prints what it is *not* proving. + +This matters most for **bug class 4** (settle ordering), whose 200ms × 10 loop has +no reflectable upstream constant, so layer 3 is its only home. Its detector is the +invariant "a tap must not consume the entire settle budget" — a full-budget tap +(~2093ms against 2000ms) means the stability loop never latched, yet the flow +still passes, so outcome parity would miss it. The evaluator is pure and +unit-tested against synthetic traces (`invariants.test.ts`); only the device run +that produces a real trace is scheduled-only. A scenario that declares an +invariant fails if the trace is missing — a detector that cannot run is a +failure, not a pass. + +## Files + +- [`jvm-harness/`](./jvm-harness) — Gradle/Kotlin generator for layers 1–2. Depends on the + published Maestro jars; reads the parser and constants directly (never + transcribed). Requires JDK 17+. +- [`corpus/`](./corpus) — flows driven through the upstream parser. + `manifest.json` (generated by [`build-manifest.mjs`](./build-manifest.mjs), do + not hand-edit) records provenance: upstream flows are vendored verbatim with + their `sha256`; authored flows fill coverage gaps (`authored/`), encode the bug + classes (`bug-classes/`), and give the never-accept-what-upstream-rejects guard + teeth (`invalid/`). To add a flow: drop the `.yaml` in, add a note to `NOTES` + in `build-manifest.mjs`, and regenerate. +- [`fixtures/`](./fixtures) — the generated, checked-in layer-1/layer-2 captures. +- [`normalize.ts`](./normalize.ts) — canonical projection shared by both engines. +- [`verify.ts`](./verify.ts) / [`verify.test.ts`](./verify.test.ts) — the deterministic verifier. +- [`expected-divergence.ts`](./expected-divergence.ts) — declared, on-the-record divergences. +- [`differential/`](./differential) — layer-3 scenarios, runner, engine-side + invariants, and `flows/` (device flows against the fixture app). +- [`fixture-seal.mjs`](./fixture-seal.mjs) — content seal shared by the generator + and the verifier. +- [`regenerate.mjs`](./regenerate.mjs) — SHA-verifies the jars, rebuilds the + corpus manifest and fixtures, and seals them. + +## Verify (per-PR, no Java) + +```sh +pnpm maestro:conformance +# node --test scripts/maestro-conformance/verify.test.ts differential/run.test.ts +``` + +The verifier fails on any **undeclared** divergence: a flow our engine parses +that upstream rejects (a conformance regression), an undeclared parse mismatch, +or an undeclared `we-reject`. Every `we-reject` must list the unsupported +command/option in [`expected-divergence.ts`](./expected-divergence.ts) — that +list is the mechanical parity backlog. + +## Regenerate (only on an upstream-pin bump) + +Heavy, manual, needs JDK 17+ (Gradle is provided by the committed wrapper): + +```sh +pnpm maestro:conformance:regenerate +``` + +This resolves the pinned jars, **verifies their SHA-256 against +`pinned-upstream.json`**, runs the harness over the corpus, and rewrites +`fixtures/`. Review the diff, then run the verifier. To bump the pin: update +`pinned-upstream.json` (version/tag/commit + the jar SHA-256s from Maven Central), +refresh the vendored corpus flows and their `manifest.json` `sha256`s, regenerate, +and reconcile any new divergences. + +## Layer 3 (device) + +```sh +pnpm maestro:conformance:differential -- --platform ios --out-dir .tmp/diff +``` + +Runs scheduled on the `conformance-differential` workflow, and on demand via +`workflow_dispatch`. `--dry-run` validates the scenario registry without a device. + +The workflow builds and installs the fixture app, verifies its bundle id, pins +the Maestro CLI to the same version as layers 1-2, and passes `--maestro` so the +flow routes through the compat engine. + +The built `.app` is **cached** (keyed on the app's sources, its dependency graph, +the iOS runtime, and the Xcode version), because building it costs ~22 minutes +versus ~6 minutes for the differential itself — 79% of the job, for an app that +changes almost never. A cache hit installs the bundle directly; a miss falls back +to the full build and repopulates. If you change anything under +`examples/test-app`, expect the next run to rebuild. + +**Investigate locally, not through CI.** A device iteration in CI is ~40 minutes; +`--only` plus a local simulator is minutes: + +```sh +pnpm test-app:install && pnpm --dir examples/test-app exec expo run:ios --configuration Release +pnpm maestro:conformance:differential -- --platform ios --only settle-after-tap --trace-root .agent-device +``` diff --git a/scripts/maestro-conformance/build-manifest.mjs b/scripts/maestro-conformance/build-manifest.mjs new file mode 100644 index 000000000..6d169dfc6 --- /dev/null +++ b/scripts/maestro-conformance/build-manifest.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node +// Rebuilds corpus/manifest.json from the flows on disk. `regenerate.mjs` runs +// this first, so adding a corpus flow is just: drop the .yaml in, add a note +// below if it needs one, and regenerate. +// +// The manifest is the corpus's provenance record: vendored upstream flows carry +// their upstream repo path and sha256 so a silently-edited flow is detectable; +// authored flows say why they exist. +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const CORPUS_DIR = path.join(HERE, 'corpus'); +const UPSTREAM_REPO_DIR = 'maestro-test/src/test/resources'; + +const NOTES = { + 'bug-classes/percent-decimal-swipe': 'Bug class 1: decimal percentage rejection at parse.', + 'bug-classes/target-swipe-missing-direction': 'Bug class 2: target swipe requires an explicit direction.', + 'bug-classes/retry-over-cap': 'Bug class 3: retry maxRetries parses verbatim; the clamp is a layer-2 vector.', + 'bug-classes/settle-after-tap': 'Bug class 4: plain tap; settle ordering is a layer-3 differential.', + 'authored/runscript': 'Coverage: runScript file command (no self-contained upstream flow).', + 'authored/runflow-main': 'Coverage: runFlow file include + provenance.', + 'authored/runflow-child': 'Include target for runflow-main (not parsed as a top-level flow).', + 'authored/doubletap': 'Coverage: doubleTapOn (upstream 101 uses the unsupported retryTapIfNoChange option).', + 'authored/scroll-until-visible': 'Coverage: scrollUntilVisible (upstream 079 uses unsupported speed/visibilityPercentage).', + 'authored/extended-wait': 'Coverage: extendedWaitUntil (upstream 042 interpolates ${TIMEOUT} from a flow env block).', + 'authored/repeat': 'Coverage: repeat.times (upstream 053 also uses the unsupported evalScript command).', + 'authored/presskey': 'Coverage: pressKey supported keys (upstream 034 exercises many unsupported keycodes).', + 'invalid/bad-swipe-direction': 'Lenient-guard: unknown SwipeDirection enum value.', + 'invalid/unknown-command': 'Lenient-guard: unknown command name (tapOn typo).', + 'invalid/malformed-selector': 'Lenient-guard: selector given as a sequence.', + 'invalid/duplicate-keys': 'Lenient-guard: duplicate YAML mapping keys.', + 'invalid/unknown-selector-field': 'Lenient-guard: unknown field inside a selector map.', + 'invalid/commands-not-a-list': 'Lenient-guard: command document is not a sequence.', +}; + +const sha256 = (file) => createHash('sha256').update(fs.readFileSync(file)).digest('hex'); + +const yamlIn = (dir) => + fs.existsSync(path.join(CORPUS_DIR, dir)) + ? fs.readdirSync(path.join(CORPUS_DIR, dir)).filter((n) => n.endsWith('.yaml')).sort() + : []; + +function entriesFor(dir, kind) { + return yamlIn(dir).map((name) => { + const stem = name.slice(0, -'.yaml'.length); + const id = `${dir}/${stem}`; + const entry = { id, file: `${dir}/${name}` }; + if (kind === 'upstream') { + entry.origin = { + kind: 'upstream', + repoPath: `${UPSTREAM_REPO_DIR}/${name}`, + sha256: sha256(path.join(CORPUS_DIR, dir, name)), + }; + } else { + entry.origin = { kind: 'authored', note: NOTES[id] ?? '' }; + } + if (dir === 'bug-classes') entry.bugClass = stem; + if (id === 'authored/runflow-child') entry.includeTargetOnly = true; + return entry; + }); +} + +export function buildManifest(pin) { + return { + description: + 'Corpus of Maestro flows driven through the upstream parser to generate layer-1 fixtures. Upstream flows are vendored verbatim from the pinned commit; sha256 records provenance. Authored flows fill coverage gaps, encode the four #1217 bug classes, and (invalid/) give the never-accept-what-upstream-rejects guard teeth. Generated by build-manifest.mjs — do not hand-edit.', + upstreamCommit: pin.commit, + flows: [ + ...entriesFor('upstream', 'upstream'), + ...entriesFor('bug-classes', 'authored'), + ...entriesFor('invalid', 'authored'), + ...entriesFor('authored', 'authored'), + ], + }; +} + +export function writeManifest(pin) { + const manifest = buildManifest(pin); + fs.writeFileSync( + path.join(CORPUS_DIR, 'manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, + ); + const missing = manifest.flows.filter((f) => f.origin.kind === 'authored' && !f.origin.note); + for (const flow of missing) { + console.warn(`warning: ${flow.id} has no note — add one to NOTES in build-manifest.mjs`); + } + return manifest; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const pin = JSON.parse(fs.readFileSync(path.join(HERE, 'pinned-upstream.json'), 'utf8')); + const manifest = writeManifest(pin); + console.log(`wrote corpus/manifest.json (${manifest.flows.length} flows)`); +} diff --git a/scripts/maestro-conformance/corpus/authored/doubletap.yaml b/scripts/maestro-conformance/corpus/authored/doubletap.yaml new file mode 100644 index 000000000..4e5c0e1ec --- /dev/null +++ b/scripts/maestro-conformance/corpus/authored/doubletap.yaml @@ -0,0 +1,3 @@ +appId: com.example.app +--- +- doubleTapOn: "Button" diff --git a/scripts/maestro-conformance/corpus/authored/extended-wait.yaml b/scripts/maestro-conformance/corpus/authored/extended-wait.yaml new file mode 100644 index 000000000..9008d1296 --- /dev/null +++ b/scripts/maestro-conformance/corpus/authored/extended-wait.yaml @@ -0,0 +1,10 @@ +appId: com.example.app +--- +- extendedWaitUntil: + visible: + id: "Item" + timeout: 1000 +- extendedWaitUntil: + notVisible: + id: "Another" + timeout: 1000 diff --git a/scripts/maestro-conformance/corpus/authored/presskey.yaml b/scripts/maestro-conformance/corpus/authored/presskey.yaml new file mode 100644 index 000000000..e7d737928 --- /dev/null +++ b/scripts/maestro-conformance/corpus/authored/presskey.yaml @@ -0,0 +1,5 @@ +appId: com.example.app +--- +- pressKey: Enter +- pressKey: Home +- pressKey: Back diff --git a/scripts/maestro-conformance/corpus/authored/repeat.yaml b/scripts/maestro-conformance/corpus/authored/repeat.yaml new file mode 100644 index 000000000..a94ff5842 --- /dev/null +++ b/scripts/maestro-conformance/corpus/authored/repeat.yaml @@ -0,0 +1,6 @@ +appId: com.example.app +--- +- repeat: + times: 3 + commands: + - tapOn: "Button" diff --git a/scripts/maestro-conformance-fixtures/runflow-child.yaml b/scripts/maestro-conformance/corpus/authored/runflow-child.yaml similarity index 66% rename from scripts/maestro-conformance-fixtures/runflow-child.yaml rename to scripts/maestro-conformance/corpus/authored/runflow-child.yaml index 6060da132..9e122a166 100644 --- a/scripts/maestro-conformance-fixtures/runflow-child.yaml +++ b/scripts/maestro-conformance/corpus/authored/runflow-child.yaml @@ -2,4 +2,4 @@ appId: com.example.include --- - launchApp - tapOn: - id: included-button + id: "included-button" diff --git a/scripts/maestro-conformance/corpus/authored/runflow-main.yaml b/scripts/maestro-conformance/corpus/authored/runflow-main.yaml new file mode 100644 index 000000000..85f2abe85 --- /dev/null +++ b/scripts/maestro-conformance/corpus/authored/runflow-main.yaml @@ -0,0 +1,5 @@ +appId: com.example.app +--- +- tapOn: "Before" +- runFlow: runflow-child.yaml +- tapOn: "After" diff --git a/scripts/maestro-conformance/corpus/authored/runscript.js b/scripts/maestro-conformance/corpus/authored/runscript.js new file mode 100644 index 000000000..17213f68c --- /dev/null +++ b/scripts/maestro-conformance/corpus/authored/runscript.js @@ -0,0 +1 @@ +output.doubled = 2 + 2; diff --git a/scripts/maestro-conformance/corpus/authored/runscript.yaml b/scripts/maestro-conformance/corpus/authored/runscript.yaml new file mode 100644 index 000000000..6b5cf51fb --- /dev/null +++ b/scripts/maestro-conformance/corpus/authored/runscript.yaml @@ -0,0 +1,3 @@ +appId: com.example.app +--- +- runScript: runscript.js diff --git a/scripts/maestro-conformance/corpus/authored/scroll-until-visible.yaml b/scripts/maestro-conformance/corpus/authored/scroll-until-visible.yaml new file mode 100644 index 000000000..c578a3b9d --- /dev/null +++ b/scripts/maestro-conformance/corpus/authored/scroll-until-visible.yaml @@ -0,0 +1,7 @@ +appId: com.example.app +--- +- scrollUntilVisible: + element: + text: "Test" + direction: DOWN + timeout: 10000 diff --git a/scripts/maestro-conformance/corpus/bug-classes/percent-decimal-swipe.yaml b/scripts/maestro-conformance/corpus/bug-classes/percent-decimal-swipe.yaml new file mode 100644 index 000000000..b02bd06d8 --- /dev/null +++ b/scripts/maestro-conformance/corpus/bug-classes/percent-decimal-swipe.yaml @@ -0,0 +1,7 @@ +# Bug class 1 (percent rounding/rejection): decimal percentages must be rejected, +# never rounded. Upstream `"50.5".toInt()` throws at parse; agent-device rejects too. +appId: com.example.app +--- +- swipe: + start: "50.5%, 50%" + end: "10%, 50%" diff --git a/scripts/maestro-conformance/corpus/bug-classes/retry-over-cap.yaml b/scripts/maestro-conformance/corpus/bug-classes/retry-over-cap.yaml new file mode 100644 index 000000000..1fe6fa092 --- /dev/null +++ b/scripts/maestro-conformance/corpus/bug-classes/retry-over-cap.yaml @@ -0,0 +1,9 @@ +# Bug class 3 (retry-cap semantics): maxRetries above the cap parses verbatim; +# the clamp to MAX_RETRIES_ALLOWED=3 is a layer-2 semantic vector, not a parse +# transform. This flow proves parse parity for the raw value. +appId: com.example.app +--- +- retry: + maxRetries: 99 + commands: + - tapOn: "Retry" diff --git a/scripts/maestro-conformance/corpus/bug-classes/settle-after-tap.yaml b/scripts/maestro-conformance/corpus/bug-classes/settle-after-tap.yaml new file mode 100644 index 000000000..05650fa66 --- /dev/null +++ b/scripts/maestro-conformance/corpus/bug-classes/settle-after-tap.yaml @@ -0,0 +1,7 @@ +# Bug class 4 (settle-loop ordering): a plain tap parses identically on both +# engines; the sleep-after vs sleep-before capture ordering and the 200ms x 10 +# loop are app-observable, verified by the layer-3 differential scenario of the +# same name (no reflectable upstream constant exists). +appId: com.example.app +--- +- tapOn: "Submit" diff --git a/scripts/maestro-conformance/corpus/bug-classes/target-swipe-missing-direction.yaml b/scripts/maestro-conformance/corpus/bug-classes/target-swipe-missing-direction.yaml new file mode 100644 index 000000000..a7398af0b --- /dev/null +++ b/scripts/maestro-conformance/corpus/bug-classes/target-swipe-missing-direction.yaml @@ -0,0 +1,7 @@ +# Bug class 2 (target-swipe direction default): a from-element swipe with no +# direction must be rejected at parse, never silently defaulted to a direction. +appId: com.example.app +--- +- swipe: + from: + id: "row" diff --git a/scripts/maestro-conformance/corpus/invalid/bad-swipe-direction.yaml b/scripts/maestro-conformance/corpus/invalid/bad-swipe-direction.yaml new file mode 100644 index 000000000..8e6769a51 --- /dev/null +++ b/scripts/maestro-conformance/corpus/invalid/bad-swipe-direction.yaml @@ -0,0 +1,5 @@ +# Upstream rejects an unknown SwipeDirection enum value at parse. +appId: com.example.app +--- +- swipe: + direction: SIDEWAYS diff --git a/scripts/maestro-conformance/corpus/invalid/commands-not-a-list.yaml b/scripts/maestro-conformance/corpus/invalid/commands-not-a-list.yaml new file mode 100644 index 000000000..ae3b78c13 --- /dev/null +++ b/scripts/maestro-conformance/corpus/invalid/commands-not-a-list.yaml @@ -0,0 +1,4 @@ +# The command document must be a sequence. +appId: com.example.app +--- +tapOn: "Button" diff --git a/scripts/maestro-conformance/corpus/invalid/duplicate-keys.yaml b/scripts/maestro-conformance/corpus/invalid/duplicate-keys.yaml new file mode 100644 index 000000000..9c9d7149b --- /dev/null +++ b/scripts/maestro-conformance/corpus/invalid/duplicate-keys.yaml @@ -0,0 +1,6 @@ +# Duplicate mapping keys are a YAML-level error. +appId: com.example.app +--- +- tapOn: + text: "A" + text: "B" diff --git a/scripts/maestro-conformance/corpus/invalid/malformed-selector.yaml b/scripts/maestro-conformance/corpus/invalid/malformed-selector.yaml new file mode 100644 index 000000000..ecf2d9aa2 --- /dev/null +++ b/scripts/maestro-conformance/corpus/invalid/malformed-selector.yaml @@ -0,0 +1,5 @@ +# Upstream rejects a selector that is a sequence rather than a scalar/map. +appId: com.example.app +--- +- tapOn: + - text: "Button" diff --git a/scripts/maestro-conformance/corpus/invalid/unknown-command.yaml b/scripts/maestro-conformance/corpus/invalid/unknown-command.yaml new file mode 100644 index 000000000..2b6b7c1a1 --- /dev/null +++ b/scripts/maestro-conformance/corpus/invalid/unknown-command.yaml @@ -0,0 +1,4 @@ +# Upstream rejects an unknown command name (typo of tapOn). +appId: com.example.app +--- +- tapOnn: "Button" diff --git a/scripts/maestro-conformance/corpus/invalid/unknown-selector-field.yaml b/scripts/maestro-conformance/corpus/invalid/unknown-selector-field.yaml new file mode 100644 index 000000000..e81afabd6 --- /dev/null +++ b/scripts/maestro-conformance/corpus/invalid/unknown-selector-field.yaml @@ -0,0 +1,6 @@ +# Upstream rejects an unknown field inside a selector map. +appId: com.example.app +--- +- tapOn: + text: "Button" + bogusField: true diff --git a/scripts/maestro-conformance/corpus/manifest.json b/scripts/maestro-conformance/corpus/manifest.json new file mode 100644 index 000000000..973a57cb4 --- /dev/null +++ b/scripts/maestro-conformance/corpus/manifest.json @@ -0,0 +1,533 @@ +{ + "description": "Corpus of Maestro flows driven through the upstream parser to generate layer-1 fixtures. Upstream flows are vendored verbatim from the pinned commit; sha256 records provenance. Authored flows fill coverage gaps, encode the four #1217 bug classes, and (invalid/) give the never-accept-what-upstream-rejects guard teeth. Generated by build-manifest.mjs — do not hand-edit.", + "upstreamCommit": "a4c7c95f5ba1884858f7e35efa6b8e0165db9448", + "flows": [ + { + "id": "upstream/001_assert_visible_by_id", + "file": "upstream/001_assert_visible_by_id.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/001_assert_visible_by_id.yaml", + "sha256": "b4202f6bf6f518a528ecff56d5c0182923e40e1df7e8cf3c8bc675e879668d32" + } + }, + { + "id": "upstream/002_assert_visible_by_text", + "file": "upstream/002_assert_visible_by_text.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/002_assert_visible_by_text.yaml", + "sha256": "6f66f19f905910de8c9ffe98cad1088457fb601e5cf9e801f6393736332f33c7" + } + }, + { + "id": "upstream/008_tap_on_element", + "file": "upstream/008_tap_on_element.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/008_tap_on_element.yaml", + "sha256": "190c54f3c4e7e6dc2eb1d3d8ccfd21d5a179d0b07f299f8526ddbc238600025d" + } + }, + { + "id": "upstream/009_skip_optional_elements", + "file": "upstream/009_skip_optional_elements.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/009_skip_optional_elements.yaml", + "sha256": "cadca0445e86092ecc603eb15827f8790c708630cb3058b83976f65d4d03bf22" + } + }, + { + "id": "upstream/010_scroll", + "file": "upstream/010_scroll.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/010_scroll.yaml", + "sha256": "7f0e2629395b89a7f5f4291f9b0bdb64f94d7351de39d3f8430ead333508ebf3" + } + }, + { + "id": "upstream/011_back_press", + "file": "upstream/011_back_press.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/011_back_press.yaml", + "sha256": "d8b9b0a7f6642e38373165dadcfcc86694929aca37a30d96bcc5fbe928dad7ed" + } + }, + { + "id": "upstream/012_input_text", + "file": "upstream/012_input_text.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/012_input_text.yaml", + "sha256": "f1650cf33e23b832e41f8ad779b8024f3c4a9210122201c420cfbc50087ae3c5" + } + }, + { + "id": "upstream/013_launch_app", + "file": "upstream/013_launch_app.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/013_launch_app.yaml", + "sha256": "072f4842fe1e83cc3d3ae709b699cbb2ac08e1978c42195195fc3dd899b391ee" + } + }, + { + "id": "upstream/014_tap_on_point", + "file": "upstream/014_tap_on_point.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/014_tap_on_point.yaml", + "sha256": "fa98bac7e0e4333870ea3541d9db697dd08f8123519efad8addf8b3e3da3bfc8" + } + }, + { + "id": "upstream/017_swipe", + "file": "upstream/017_swipe.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/017_swipe.yaml", + "sha256": "a1fec7900bf845045d79661bc0302b03b0a52002d2ad5ba39122f30d64722f6a" + } + }, + { + "id": "upstream/021_launch_app_with_clear_state", + "file": "upstream/021_launch_app_with_clear_state.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/021_launch_app_with_clear_state.yaml", + "sha256": "0f92f311675e2936ebe7064de6c9a3460e7f26e860740a19edb5da4a9f2febba" + } + }, + { + "id": "upstream/026_assert_not_visible", + "file": "upstream/026_assert_not_visible.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/026_assert_not_visible.yaml", + "sha256": "31dfb8e7c00c96dc948cca2ccaa9523dde1d17d204d544b78c3159dddcae3076" + } + }, + { + "id": "upstream/027_open_link", + "file": "upstream/027_open_link.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/027_open_link.yaml", + "sha256": "a394719a3700c17eb85c6436445ac2549cf0f4650ffa5e887dfb51ccedae4b77" + } + }, + { + "id": "upstream/029_long_press_on_element", + "file": "upstream/029_long_press_on_element.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/029_long_press_on_element.yaml", + "sha256": "952c68c4c38ae64aede8ff0098fe698ce5f62a2c9e9f633f17ea3cd38b6b3f55" + } + }, + { + "id": "upstream/032_element_index", + "file": "upstream/032_element_index.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/032_element_index.yaml", + "sha256": "4a793304c59aeacbcf6b4dd0bcbc1a0f2fb6b67105975b57d72f1617dedbb4e3" + } + }, + { + "id": "upstream/034_press_key", + "file": "upstream/034_press_key.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/034_press_key.yaml", + "sha256": "5a8885d87e7232e179ae99774fea89ff38a62294f6aab3b2564eefa1b38a8a82" + } + }, + { + "id": "upstream/036_erase_text", + "file": "upstream/036_erase_text.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/036_erase_text.yaml", + "sha256": "e08ce0ce034865ee3dc9b5cc7576a89b3ae31bed32c6d22f32a462041605f999" + } + }, + { + "id": "upstream/039_hide_keyboard", + "file": "upstream/039_hide_keyboard.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/039_hide_keyboard.yaml", + "sha256": "d3fc657a45e00cd6c3d477907ac1890a4262dc2e4144fbb6128b2bae560ac21a" + } + }, + { + "id": "upstream/041_take_screenshot", + "file": "upstream/041_take_screenshot.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/041_take_screenshot.yaml", + "sha256": "cd7ec560a87a84843f390ce5ee970e8fe89d913a444e2a333b5951b945f93f73" + } + }, + { + "id": "upstream/042_extended_wait", + "file": "upstream/042_extended_wait.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/042_extended_wait.yaml", + "sha256": "bf2636106ad49bf21094be2173cd7e9afc98362536ff7f4b2a23c6a42f5cec5f" + } + }, + { + "id": "upstream/043_stop_app", + "file": "upstream/043_stop_app.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/043_stop_app.yaml", + "sha256": "0ff282185ca1975d4e542a812893473f9c6341926e02f69ce02bc6b39134c819" + } + }, + { + "id": "upstream/045_clear_keychain", + "file": "upstream/045_clear_keychain.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/045_clear_keychain.yaml", + "sha256": "ef690d7423bb6a88873429fdd069290eb8ced067c552bcbd3c1f3e6e6f35c131" + } + }, + { + "id": "upstream/051_set_location", + "file": "upstream/051_set_location.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/051_set_location.yaml", + "sha256": "3d951d98e1574174bf759d6cf82eed69225546d585daab9d63f78af941cc0f92" + } + }, + { + "id": "upstream/053_repeat_times", + "file": "upstream/053_repeat_times.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/053_repeat_times.yaml", + "sha256": "5b8dac21447ead97ac7ea89574ccc54f4952203eea89f0c4a77ecf4bf03c7d87" + } + }, + { + "id": "upstream/059_directional_swipe_command", + "file": "upstream/059_directional_swipe_command.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/059_directional_swipe_command.yaml", + "sha256": "d89154c8a7f25144bafbabcd2a802e1619653611579f4bfb03a815c82f11c18d" + } + }, + { + "id": "upstream/061_launchApp_withoutStopping", + "file": "upstream/061_launchApp_withoutStopping.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/061_launchApp_withoutStopping.yaml", + "sha256": "ccb189359bab48f064f51b20a857c185050af0d392db7b63d9131cb8cf979819" + } + }, + { + "id": "upstream/062_copy_paste_text", + "file": "upstream/062_copy_paste_text.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/062_copy_paste_text.yaml", + "sha256": "1ead5d9b237d6f0915eeb00b05eea52a65ad9d083df51a880cc62513399c9fad" + } + }, + { + "id": "upstream/067_assertTrue_pass", + "file": "upstream/067_assertTrue_pass.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/067_assertTrue_pass.yaml", + "sha256": "cd602d4398468cdbdf40edebca0562d6c1e6f9e3c7d5d506975f486c9d33f32b" + } + }, + { + "id": "upstream/069_wait_for_animation_to_end", + "file": "upstream/069_wait_for_animation_to_end.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/069_wait_for_animation_to_end.yaml", + "sha256": "aa5870c7c9db3445bad9f1b6c08f29d0523d9c3d7c33cc23b12c5bc76007032b" + } + }, + { + "id": "upstream/074_directional_swipe_element", + "file": "upstream/074_directional_swipe_element.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/074_directional_swipe_element.yaml", + "sha256": "63b71f6ed33a619b391fb149f0a1ee3a47ec00f254a8b5135dcc47c8ec645b0f" + } + }, + { + "id": "upstream/076_optional_assertion", + "file": "upstream/076_optional_assertion.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/076_optional_assertion.yaml", + "sha256": "409fd51ffbf3cd1e62d6d6d8ac837c47bdc3d1c8da73ac7c342bc093e5c31851" + } + }, + { + "id": "upstream/078_swipe_relative", + "file": "upstream/078_swipe_relative.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/078_swipe_relative.yaml", + "sha256": "a4a0430ba0cb0df1c8f3d606d2a9a73b33981bfd56069d144537e49623b1524a" + } + }, + { + "id": "upstream/079_scroll_until_visible", + "file": "upstream/079_scroll_until_visible.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/079_scroll_until_visible.yaml", + "sha256": "7e3709703ff1e6056c8e3c31d503ea1a9841536a2113342ddaa190725a648124" + } + }, + { + "id": "upstream/090_travel", + "file": "upstream/090_travel.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/090_travel.yaml", + "sha256": "bd50cc7cbf20832337e526c3617851bc963bab8b46afae30e648e7336d490c18" + } + }, + { + "id": "upstream/094_runFlow_inline", + "file": "upstream/094_runFlow_inline.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/094_runFlow_inline.yaml", + "sha256": "3392332bd22731a8ae5c30bf59039926799acc960a95d9588424798dc4d881f6" + } + }, + { + "id": "upstream/100_tapOn_multiple_times", + "file": "upstream/100_tapOn_multiple_times.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/100_tapOn_multiple_times.yaml", + "sha256": "d5a3dea0decd9be14ee28ac1fc390ff659d5671f66da1d6e5cd1d9aef4bce183" + } + }, + { + "id": "upstream/101_doubleTapOn", + "file": "upstream/101_doubleTapOn.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/101_doubleTapOn.yaml", + "sha256": "f4d73db5712a8e8f9b7ddda5ecbb33b80898d3b9fe03e567dc618d98ad783326" + } + }, + { + "id": "upstream/114_child_of_selector", + "file": "upstream/114_child_of_selector.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/114_child_of_selector.yaml", + "sha256": "f03587d907b7ad35ab677fe1903d10c3059e80d9ecdd9cbbf17fbf6a1dd0a552" + } + }, + { + "id": "upstream/116_kill_app", + "file": "upstream/116_kill_app.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/116_kill_app.yaml", + "sha256": "5caf5c05d867a6ac31e289837e3fea170adcd90f5fbfd9be5a497c437fc72b6f" + } + }, + { + "id": "upstream/119_retry_commands", + "file": "upstream/119_retry_commands.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/119_retry_commands.yaml", + "sha256": "56c43c2f13aa1873c3a49a418efc0942c446534e6444d2e8f657b252fcc97bae" + } + }, + { + "id": "upstream/120_tap_on_element_retryTapIfNoChange", + "file": "upstream/120_tap_on_element_retryTapIfNoChange.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/120_tap_on_element_retryTapIfNoChange.yaml", + "sha256": "a8889dc193f739d208ea2694059d0686288d0b2f61af997c250bb29d5bbcb748" + } + }, + { + "id": "upstream/131_setPermissions", + "file": "upstream/131_setPermissions.yaml", + "origin": { + "kind": "upstream", + "repoPath": "maestro-test/src/test/resources/131_setPermissions.yaml", + "sha256": "874edbd2691ef5803b56477d44cb308abc69da8ef844e5a64a434d356847004f" + } + }, + { + "id": "bug-classes/percent-decimal-swipe", + "file": "bug-classes/percent-decimal-swipe.yaml", + "origin": { + "kind": "authored", + "note": "Bug class 1: decimal percentage rejection at parse." + }, + "bugClass": "percent-decimal-swipe" + }, + { + "id": "bug-classes/retry-over-cap", + "file": "bug-classes/retry-over-cap.yaml", + "origin": { + "kind": "authored", + "note": "Bug class 3: retry maxRetries parses verbatim; the clamp is a layer-2 vector." + }, + "bugClass": "retry-over-cap" + }, + { + "id": "bug-classes/settle-after-tap", + "file": "bug-classes/settle-after-tap.yaml", + "origin": { + "kind": "authored", + "note": "Bug class 4: plain tap; settle ordering is a layer-3 differential." + }, + "bugClass": "settle-after-tap" + }, + { + "id": "bug-classes/target-swipe-missing-direction", + "file": "bug-classes/target-swipe-missing-direction.yaml", + "origin": { + "kind": "authored", + "note": "Bug class 2: target swipe requires an explicit direction." + }, + "bugClass": "target-swipe-missing-direction" + }, + { + "id": "invalid/bad-swipe-direction", + "file": "invalid/bad-swipe-direction.yaml", + "origin": { + "kind": "authored", + "note": "Lenient-guard: unknown SwipeDirection enum value." + } + }, + { + "id": "invalid/commands-not-a-list", + "file": "invalid/commands-not-a-list.yaml", + "origin": { + "kind": "authored", + "note": "Lenient-guard: command document is not a sequence." + } + }, + { + "id": "invalid/duplicate-keys", + "file": "invalid/duplicate-keys.yaml", + "origin": { + "kind": "authored", + "note": "Lenient-guard: duplicate YAML mapping keys." + } + }, + { + "id": "invalid/malformed-selector", + "file": "invalid/malformed-selector.yaml", + "origin": { + "kind": "authored", + "note": "Lenient-guard: selector given as a sequence." + } + }, + { + "id": "invalid/unknown-command", + "file": "invalid/unknown-command.yaml", + "origin": { + "kind": "authored", + "note": "Lenient-guard: unknown command name (tapOn typo)." + } + }, + { + "id": "invalid/unknown-selector-field", + "file": "invalid/unknown-selector-field.yaml", + "origin": { + "kind": "authored", + "note": "Lenient-guard: unknown field inside a selector map." + } + }, + { + "id": "authored/doubletap", + "file": "authored/doubletap.yaml", + "origin": { + "kind": "authored", + "note": "Coverage: doubleTapOn (upstream 101 uses the unsupported retryTapIfNoChange option)." + } + }, + { + "id": "authored/extended-wait", + "file": "authored/extended-wait.yaml", + "origin": { + "kind": "authored", + "note": "Coverage: extendedWaitUntil (upstream 042 interpolates ${TIMEOUT} from a flow env block)." + } + }, + { + "id": "authored/presskey", + "file": "authored/presskey.yaml", + "origin": { + "kind": "authored", + "note": "Coverage: pressKey supported keys (upstream 034 exercises many unsupported keycodes)." + } + }, + { + "id": "authored/repeat", + "file": "authored/repeat.yaml", + "origin": { + "kind": "authored", + "note": "Coverage: repeat.times (upstream 053 also uses the unsupported evalScript command)." + } + }, + { + "id": "authored/runflow-child", + "file": "authored/runflow-child.yaml", + "origin": { + "kind": "authored", + "note": "Include target for runflow-main (not parsed as a top-level flow)." + }, + "includeTargetOnly": true + }, + { + "id": "authored/runflow-main", + "file": "authored/runflow-main.yaml", + "origin": { + "kind": "authored", + "note": "Coverage: runFlow file include + provenance." + } + }, + { + "id": "authored/runscript", + "file": "authored/runscript.yaml", + "origin": { + "kind": "authored", + "note": "Coverage: runScript file command (no self-contained upstream flow)." + } + }, + { + "id": "authored/scroll-until-visible", + "file": "authored/scroll-until-visible.yaml", + "origin": { + "kind": "authored", + "note": "Coverage: scrollUntilVisible (upstream 079 uses unsupported speed/visibilityPercentage)." + } + } + ] +} diff --git a/scripts/maestro-conformance/corpus/upstream/001_assert_visible_by_id.yaml b/scripts/maestro-conformance/corpus/upstream/001_assert_visible_by_id.yaml new file mode 100644 index 000000000..5552e4dba --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/001_assert_visible_by_id.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- assertVisible: + id: "element_id" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/002_assert_visible_by_text.yaml b/scripts/maestro-conformance/corpus/upstream/002_assert_visible_by_text.yaml new file mode 100644 index 000000000..4d05bf058 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/002_assert_visible_by_text.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- assertVisible: + text: "Element Text" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/008_tap_on_element.yaml b/scripts/maestro-conformance/corpus/upstream/008_tap_on_element.yaml new file mode 100644 index 000000000..259034234 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/008_tap_on_element.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- tapOn: + text: ".*button.*" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/009_skip_optional_elements.yaml b/scripts/maestro-conformance/corpus/upstream/009_skip_optional_elements.yaml new file mode 100644 index 000000000..66ef9c30d --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/009_skip_optional_elements.yaml @@ -0,0 +1,8 @@ +appId: com.example.app +--- +- tapOn: + text: "Optional Element" + optional: true +- assertVisible: + text: "Non Optional" + optional: false \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/010_scroll.yaml b/scripts/maestro-conformance/corpus/upstream/010_scroll.yaml new file mode 100644 index 000000000..bd91ecc5c --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/010_scroll.yaml @@ -0,0 +1,3 @@ +appId: com.example.app +--- +- scroll \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/011_back_press.yaml b/scripts/maestro-conformance/corpus/upstream/011_back_press.yaml new file mode 100644 index 000000000..cd7d0c53d --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/011_back_press.yaml @@ -0,0 +1,3 @@ +appId: com.example.app +--- +- back \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/012_input_text.yaml b/scripts/maestro-conformance/corpus/upstream/012_input_text.yaml new file mode 100644 index 000000000..2ba2ee5e5 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/012_input_text.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- inputText: "Hello World" +- inputText: user@example.com \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/013_launch_app.yaml b/scripts/maestro-conformance/corpus/upstream/013_launch_app.yaml new file mode 100644 index 000000000..e98c0c42b --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/013_launch_app.yaml @@ -0,0 +1,3 @@ +appId: com.example.app +--- +- launchApp \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/014_tap_on_point.yaml b/scripts/maestro-conformance/corpus/upstream/014_tap_on_point.yaml new file mode 100644 index 000000000..061a370ab --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/014_tap_on_point.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- tapOn: + point: 100,200 \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/017_swipe.yaml b/scripts/maestro-conformance/corpus/upstream/017_swipe.yaml new file mode 100644 index 000000000..c358ecb69 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/017_swipe.yaml @@ -0,0 +1,6 @@ +appId: com.example.app +--- +- swipe: + start: 100,500 + end: 100,200 + duration: 3000 \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/021_launch_app_with_clear_state.yaml b/scripts/maestro-conformance/corpus/upstream/021_launch_app_with_clear_state.yaml new file mode 100644 index 000000000..46ce97e75 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/021_launch_app_with_clear_state.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- launchApp: + clearState: true \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/026_assert_not_visible.yaml b/scripts/maestro-conformance/corpus/upstream/026_assert_not_visible.yaml new file mode 100644 index 000000000..42cc23970 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/026_assert_not_visible.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- assertNotVisible: + id: "element_id" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/027_open_link.yaml b/scripts/maestro-conformance/corpus/upstream/027_open_link.yaml new file mode 100644 index 000000000..b5c8d1b6b --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/027_open_link.yaml @@ -0,0 +1,3 @@ +appId: com.example.app +--- +- openLink: https://example.com \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/029_long_press_on_element.yaml b/scripts/maestro-conformance/corpus/upstream/029_long_press_on_element.yaml new file mode 100644 index 000000000..440f73d7f --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/029_long_press_on_element.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- longPressOn: + text: ".*button.*" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/032_element_index.yaml b/scripts/maestro-conformance/corpus/upstream/032_element_index.yaml new file mode 100644 index 000000000..7e7b2abf3 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/032_element_index.yaml @@ -0,0 +1,10 @@ +appId: com.example.app +--- +- tapOn: + text: Item.* + index: 0 + retryTapIfNoChange: false +- tapOn: + text: Item.* + index: ${0 + 1} + retryTapIfNoChange: false \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/034_press_key.yaml b/scripts/maestro-conformance/corpus/upstream/034_press_key.yaml new file mode 100644 index 000000000..8d72550e7 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/034_press_key.yaml @@ -0,0 +1,32 @@ +appId: com.example.app +--- +- pressKey: Enter +- pressKey: Backspace +- pressKey: Home +- pressKey: back +- pressKey: Volume Up +- pressKey: volume Down +- pressKey: lOcK +- pressKey: remote dpad up +- pressKey: Remote dpad Down +- pressKey: Remote Dpad Left +- pressKey: Remote Dpad Right +- pressKey: remote dpad Center +- pressKey: rEmote Media plAy Pause +- pressKey: RemOte MediA StoP +- pressKey: remote Media NeXt +- pressKey: Remote Media Previous +- pressKey: REMOTE MEDIA REWIND +- pressKey: Remote Media Fast Forward +- pressKey: power +- pressKey: Tab +- pressKey: Remote System Navigation Up +- pressKey: Remote System Navigation Down +- pressKey: Remote Button A +- pressKey: Remote Button B +- pressKey: Remote Menu +- pressKey: TV Input +- pressKey: TV Input HDMI 1 +- pressKey: TV Input HDMI 2 +- pressKey: TV Input HDMI 3 + diff --git a/scripts/maestro-conformance/corpus/upstream/036_erase_text.yaml b/scripts/maestro-conformance/corpus/upstream/036_erase_text.yaml new file mode 100644 index 000000000..6fbef1406 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/036_erase_text.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- inputText: Hello World +- eraseText: 6 diff --git a/scripts/maestro-conformance/corpus/upstream/039_hide_keyboard.yaml b/scripts/maestro-conformance/corpus/upstream/039_hide_keyboard.yaml new file mode 100644 index 000000000..93ab4f21e --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/039_hide_keyboard.yaml @@ -0,0 +1,3 @@ +appId: com.example.app +--- +- hideKeyboard \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/041_take_screenshot.yaml b/scripts/maestro-conformance/corpus/upstream/041_take_screenshot.yaml new file mode 100644 index 000000000..1d30665bb --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/041_take_screenshot.yaml @@ -0,0 +1,3 @@ +appId: com.example.app +--- +- takeScreenshot: ${MAESTRO_FILENAME}_with_filename diff --git a/scripts/maestro-conformance/corpus/upstream/042_extended_wait.yaml b/scripts/maestro-conformance/corpus/upstream/042_extended_wait.yaml new file mode 100644 index 000000000..dd58226fa --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/042_extended_wait.yaml @@ -0,0 +1,12 @@ +appId: com.example.app +env: + TIMEOUT: 1000 +--- +- extendedWaitUntil: + visible: Item + timeout: ${TIMEOUT} +- extendedWaitUntil: + notVisible: Another item + timeout: 1000 +- extendedWaitUntil: + visible: Item diff --git a/scripts/maestro-conformance/corpus/upstream/043_stop_app.yaml b/scripts/maestro-conformance/corpus/upstream/043_stop_app.yaml new file mode 100644 index 000000000..e40576acf --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/043_stop_app.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- stopApp +- stopApp: another.app diff --git a/scripts/maestro-conformance/corpus/upstream/045_clear_keychain.yaml b/scripts/maestro-conformance/corpus/upstream/045_clear_keychain.yaml new file mode 100644 index 000000000..58e323fa8 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/045_clear_keychain.yaml @@ -0,0 +1,6 @@ +appId: com.example.app +--- +- clearKeychain +- launchApp: + appId: com.example.app + clearKeychain: true diff --git a/scripts/maestro-conformance/corpus/upstream/051_set_location.yaml b/scripts/maestro-conformance/corpus/upstream/051_set_location.yaml new file mode 100644 index 000000000..9045175c1 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/051_set_location.yaml @@ -0,0 +1,7 @@ +appId: com.example.app +--- +- launchApp: + appId: com.example.app +- setLocation: + latitude: 12.5266 + longitude: 78.2150 diff --git a/scripts/maestro-conformance/corpus/upstream/053_repeat_times.yaml b/scripts/maestro-conformance/corpus/upstream/053_repeat_times.yaml new file mode 100644 index 000000000..e2859cb86 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/053_repeat_times.yaml @@ -0,0 +1,13 @@ +appId: com.other.app +--- +- repeat: + times: 3 + commands: + - tapOn: Button +- assertVisible: "3" +- evalScript: ${output.list = [1, 2, 3]} +- repeat: + times: ${output.list.length} + commands: + - tapOn: Button +- assertVisible: "6" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/059_directional_swipe_command.yaml b/scripts/maestro-conformance/corpus/upstream/059_directional_swipe_command.yaml new file mode 100644 index 000000000..c548e5468 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/059_directional_swipe_command.yaml @@ -0,0 +1,5 @@ +appId: com.example.app +--- +- swipe: + direction: RIGHT + duration: 500 \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/061_launchApp_withoutStopping.yaml b/scripts/maestro-conformance/corpus/upstream/061_launchApp_withoutStopping.yaml new file mode 100644 index 000000000..11e1a8348 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/061_launchApp_withoutStopping.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- launchApp: + stopApp: false \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/062_copy_paste_text.yaml b/scripts/maestro-conformance/corpus/upstream/062_copy_paste_text.yaml new file mode 100644 index 000000000..4c05431f7 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/062_copy_paste_text.yaml @@ -0,0 +1,5 @@ +appId: com.example.app +--- +- copyTextFrom: + id: "myId" +- pasteText diff --git a/scripts/maestro-conformance/corpus/upstream/067_assertTrue_pass.yaml b/scripts/maestro-conformance/corpus/upstream/067_assertTrue_pass.yaml new file mode 100644 index 000000000..abad0c509 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/067_assertTrue_pass.yaml @@ -0,0 +1,3 @@ +appId: com.example.app +--- +- assertTrue: ${1+1} \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/069_wait_for_animation_to_end.yaml b/scripts/maestro-conformance/corpus/upstream/069_wait_for_animation_to_end.yaml new file mode 100644 index 000000000..e8832c37a --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/069_wait_for_animation_to_end.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- waitForAnimationToEnd: + timeout: 500 \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/074_directional_swipe_element.yaml b/scripts/maestro-conformance/corpus/upstream/074_directional_swipe_element.yaml new file mode 100644 index 000000000..604241ce9 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/074_directional_swipe_element.yaml @@ -0,0 +1,6 @@ +appId: com.example.app +--- +- swipe: + direction: RIGHT + from: + text: "swiping element" diff --git a/scripts/maestro-conformance/corpus/upstream/076_optional_assertion.yaml b/scripts/maestro-conformance/corpus/upstream/076_optional_assertion.yaml new file mode 100644 index 000000000..50329a922 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/076_optional_assertion.yaml @@ -0,0 +1,18 @@ +appId: com.example.app +--- +- scrollUntilVisible: + timeout: 1 + element: + id: "not_found" + optional: true +- assertTrue: + condition: "false" + optional: true +- extendedWaitUntil: + visible: + id: "not_found" + timeout: 1 + optional: true +- assertVisible: + text: "Button" + optional: true diff --git a/scripts/maestro-conformance/corpus/upstream/078_swipe_relative.yaml b/scripts/maestro-conformance/corpus/upstream/078_swipe_relative.yaml new file mode 100644 index 000000000..18d45d3f8 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/078_swipe_relative.yaml @@ -0,0 +1,6 @@ +appId: com.example.app +--- +- swipe: + start: "50%,30%" + end: "50%,60%" + duration: 3000 \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/079_scroll_until_visible.yaml b/scripts/maestro-conformance/corpus/upstream/079_scroll_until_visible.yaml new file mode 100644 index 000000000..539b380e3 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/079_scroll_until_visible.yaml @@ -0,0 +1,9 @@ +appId: com.example.app +--- +- scrollUntilVisible: + element: + text: "Test" + speed: 100 + visibilityPercentage: 100 + direction: DOWN + timeout: 10 \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/090_travel.yaml b/scripts/maestro-conformance/corpus/upstream/090_travel.yaml new file mode 100644 index 000000000..bdacda629 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/090_travel.yaml @@ -0,0 +1,9 @@ +appId: com.example.app +--- +- travel: + points: + - 0.0,0.0 + - 0.1,0.0 + - 0.1,0.1 + - 0.0,0.1 + speed: 7900 # 7.9 km/s aka orbital velocity diff --git a/scripts/maestro-conformance/corpus/upstream/094_runFlow_inline.yaml b/scripts/maestro-conformance/corpus/upstream/094_runFlow_inline.yaml new file mode 100644 index 000000000..252ab4ef6 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/094_runFlow_inline.yaml @@ -0,0 +1,7 @@ +appId: com.example.app +--- +- runFlow: + env: + INNER_ENV: Inner Parameter + commands: + - inputText: ${INNER_ENV} diff --git a/scripts/maestro-conformance/corpus/upstream/100_tapOn_multiple_times.yaml b/scripts/maestro-conformance/corpus/upstream/100_tapOn_multiple_times.yaml new file mode 100644 index 000000000..a3e6a07ac --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/100_tapOn_multiple_times.yaml @@ -0,0 +1,7 @@ +appId: com.other.app +--- +- tapOn: + text: Button + repeat: 3 + delay: 1 + retryTapIfNoChange: false diff --git a/scripts/maestro-conformance/corpus/upstream/101_doubleTapOn.yaml b/scripts/maestro-conformance/corpus/upstream/101_doubleTapOn.yaml new file mode 100644 index 000000000..a69f5638b --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/101_doubleTapOn.yaml @@ -0,0 +1,5 @@ +appId: com.other.app +--- +- doubleTapOn: + text: Button + retryTapIfNoChange: false diff --git a/scripts/maestro-conformance/corpus/upstream/114_child_of_selector.yaml b/scripts/maestro-conformance/corpus/upstream/114_child_of_selector.yaml new file mode 100644 index 000000000..3ab7d1906 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/114_child_of_selector.yaml @@ -0,0 +1,10 @@ +appId: com.example.app +--- +- assertVisible: + text: "child_id" + childOf: + text: "parent_id_1" +- assertNotVisible: + text: "child_id" + childOf: + text: "parent_id_3" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/116_kill_app.yaml b/scripts/maestro-conformance/corpus/upstream/116_kill_app.yaml new file mode 100644 index 000000000..52d9f89c9 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/116_kill_app.yaml @@ -0,0 +1,4 @@ +appId: com.example.app +--- +- killApp +- killApp: another.app diff --git a/scripts/maestro-conformance/corpus/upstream/119_retry_commands.yaml b/scripts/maestro-conformance/corpus/upstream/119_retry_commands.yaml new file mode 100644 index 000000000..a2fd21643 --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/119_retry_commands.yaml @@ -0,0 +1,10 @@ +appId: com.other.app +--- +- retry: + maxRetries: 3 + commands: + - scroll + - tapOn: + text: Button + waitToSettleTimeoutMs: 40 + - scroll diff --git a/scripts/maestro-conformance/corpus/upstream/120_tap_on_element_retryTapIfNoChange.yaml b/scripts/maestro-conformance/corpus/upstream/120_tap_on_element_retryTapIfNoChange.yaml new file mode 100644 index 000000000..b6b57ebec --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/120_tap_on_element_retryTapIfNoChange.yaml @@ -0,0 +1,5 @@ +appId: com.example.app +--- +- tapOn: + text: ".*button.*" + retryTapIfNoChange: true \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/131_setPermissions.yaml b/scripts/maestro-conformance/corpus/upstream/131_setPermissions.yaml new file mode 100644 index 000000000..483fc1a5c --- /dev/null +++ b/scripts/maestro-conformance/corpus/upstream/131_setPermissions.yaml @@ -0,0 +1,6 @@ +appId: com.example.app +--- +- setPermissions: + permissions: + all: deny + notifications: unset \ No newline at end of file diff --git a/scripts/maestro-conformance/differential/flows/optional-warned-not-failed.yaml b/scripts/maestro-conformance/differential/flows/optional-warned-not-failed.yaml new file mode 100644 index 000000000..b0848feef --- /dev/null +++ b/scripts/maestro-conformance/differential/flows/optional-warned-not-failed.yaml @@ -0,0 +1,12 @@ +# Layer-3 device flow. An optional assertion on an element that never exists must +# be downgraded to a warning and let the flow finish, on both engines. A +# failed-instead-of-warned classification flips the exit code, so outcome parity +# genuinely proves this one. +appId: com.callstack.agentdevicelab +--- +- launchApp: + clearState: true +- assertVisible: + id: this-element-never-exists + optional: true +- assertVisible: Agent Device Tester diff --git a/scripts/maestro-conformance/differential/flows/percent-swipe.yaml b/scripts/maestro-conformance/differential/flows/percent-swipe.yaml new file mode 100644 index 000000000..7a1172a5f --- /dev/null +++ b/scripts/maestro-conformance/differential/flows/percent-swipe.yaml @@ -0,0 +1,12 @@ +# Layer-3 device flow. Percentage swipe endpoints must drive the same gesture on +# both engines. Deliberately stays on the scrollable home screen: no navigation, +# no off-screen controls, so the scenario tests the swipe and nothing else. +appId: com.callstack.agentdevicelab +--- +- launchApp: + clearState: true +- assertVisible: Agent Device Tester +- swipe: + start: 50%, 75% + end: 50%, 35% + duration: 300 diff --git a/scripts/maestro-conformance/differential/flows/settle-after-tap.yaml b/scripts/maestro-conformance/differential/flows/settle-after-tap.yaml new file mode 100644 index 000000000..2934a1ab9 --- /dev/null +++ b/scripts/maestro-conformance/differential/flows/settle-after-tap.yaml @@ -0,0 +1,16 @@ +# Layer-3 device flow (bug class 4). A real tap on a real control of the lab app, +# so the post-tap settle loop actually runs and the timing invariant is meaningful. +# home-open-form sits below the fold, so scroll it into view first — the app's own +# helper flow (examples/test-app/maestro/helpers/open-checkout-form.yaml) does +# exactly this; tapping it directly fails with "element not found". +appId: com.callstack.agentdevicelab +--- +- launchApp: + clearState: true +- assertVisible: Agent Device Tester +- scrollUntilVisible: + element: + id: home-open-form +- tapOn: + id: home-open-form +- assertVisible: Checkout form diff --git a/scripts/maestro-conformance/differential/flows/tap-retry-if-no-change.yaml b/scripts/maestro-conformance/differential/flows/tap-retry-if-no-change.yaml new file mode 100644 index 000000000..70c73821d --- /dev/null +++ b/scripts/maestro-conformance/differential/flows/tap-retry-if-no-change.yaml @@ -0,0 +1,15 @@ +# Layer-3 device flow. Taps the app's static title, which is NOT interactive, so +# the screen cannot change and the retryIfNoChange path is forced to run. A tap on +# a navigating control (e.g. home-open-form) succeeds on the first attempt and +# never exercises retry at all — the scenario would pass while proving nothing. +# The scenario asserts tapRetries >= 1 from the trace, so a retry regression is +# caught rather than assumed. +appId: com.callstack.agentdevicelab +--- +- launchApp: + clearState: true +- assertVisible: Agent Device Tester +- tapOn: + text: Agent Device Tester + retryTapIfNoChange: true +- assertVisible: Agent Device Tester diff --git a/scripts/maestro-conformance/differential/invariants.test.ts b/scripts/maestro-conformance/differential/invariants.test.ts new file mode 100644 index 000000000..bd5fdfaed --- /dev/null +++ b/scripts/maestro-conformance/differential/invariants.test.ts @@ -0,0 +1,154 @@ +// Device-free tests for the engine-side invariant evaluator. The evaluator is +// the bug-class-4 detector, so its logic is verified here against synthetic +// traces rather than only on the scheduled device run. +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS } from '../../../src/compat/maestro/compatibility-policy.ts'; +import { DIFFERENTIAL_SCENARIOS } from './scenarios.ts'; +import { type Invariant, evaluateInvariant, readTrace } from './invariants.ts'; + +const SETTLE_INVARIANT: Invariant = { + kind: 'stepDurationBelow', + command: 'tapOn', + maxMs: MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS, + because: 'test', +}; + +const stop = (command: string, durationMs: number, step = 1) => ({ + type: 'replay_action_stop', + step, + command, + ok: true, + durationMs, +}); + +test('a tap that latches early holds the settle invariant', () => { + // Healthy: Android ~350ms, iOS ~800-1100ms — well under the 2000ms budget. + const result = evaluateInvariant([stop('tapOn', 350)], SETTLE_INVARIANT); + assert.equal(result.status, 'held'); +}); + +test('a tap that burns the full settle budget violates the invariant (bug class 4)', () => { + // The regression signature: ~2093ms against a 200ms x 10 budget — the + // stability loop never latched, yet the flow still passes, so outcome parity + // would have missed it. + const result = evaluateInvariant([stop('tapOn', 2093)], SETTLE_INVARIANT); + assert.equal(result.status, 'violated'); + assert.match(result.detail, /2093ms/); +}); + +test('the invariant reports the slowest matching step, not the first', () => { + const result = evaluateInvariant([stop('tapOn', 300, 1), stop('tapOn', 2117, 2)], SETTLE_INVARIANT); + assert.equal(result.status, 'violated'); + assert.match(result.detail, /2117ms/); +}); + +test('a trace with no matching step reports no-data rather than passing silently', () => { + const result = evaluateInvariant([stop('swipe', 400)], SETTLE_INVARIANT); + assert.equal(result.status, 'no-data'); +}); + +test('start events and steps without a duration are ignored', () => { + const events = [ + { type: 'replay_action_start', step: 1, command: 'tapOn' }, + { type: 'replay_action_stop', step: 1, command: 'tapOn' }, + ]; + assert.equal(evaluateInvariant(events, SETTLE_INVARIANT).status, 'no-data'); +}); + +test('readTrace parses ndjson and skips blank/corrupt lines', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'conformance-trace-')); + const file = path.join(dir, 'replay-timing.ndjson'); + fs.writeFileSync(file, `${JSON.stringify(stop('tapOn', 350))}\n\nnot-json\n`); + try { + const events = readTrace(file); + assert.equal(events.length, 1); + assert.equal(events[0]?.command, 'tapOn'); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('readTrace on a missing file returns no events', () => { + assert.deepEqual(readTrace('/nonexistent/replay-timing.ndjson'), []); +}); + +test('bug class 4 has a machine-checkable invariant, not just outcome parity', () => { + const settle = DIFFERENTIAL_SCENARIOS.find((scenario) => scenario.bugClass === 4); + const invariant = settle?.engineInvariants?.[0]; + assert.ok(invariant, 'settle scenario must carry an engine-side invariant'); + assert.equal(invariant?.kind, 'stepDurationBelow'); + assert.equal( + invariant?.kind === 'stepDurationBelow' ? invariant.maxMs : undefined, + MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS, + ); +}); + +// --- metricAtLeast: proves a code path actually ran, not just that it passed --- + +const RETRY_INVARIANT: Invariant = { + kind: 'metricAtLeast', + command: 'tapOn', + metric: 'tapRetries', + min: 1, + because: 'test', +}; + +const stopWithMetrics = (command: string, metrics: Record, step = 1) => ({ + type: 'replay_action_stop', + step, + command, + ok: true, + durationMs: 100, + resultTiming: metrics, +}); + +test('a tap that actually retried holds the retry invariant', () => { + const result = evaluateInvariant([stopWithMetrics('tapOn', { tapRetries: 1 })], RETRY_INVARIANT); + assert.equal(result.status, 'held'); +}); + +// The exact vacuity that made this scenario worthless before: the tap succeeded +// first try (navigating control), so retry never ran — yet the flow passed. +test('a tap that never retried violates the invariant (catches a vacuous scenario)', () => { + const result = evaluateInvariant([stopWithMetrics('tapOn', { tapRetries: 0 })], RETRY_INVARIANT); + assert.equal(result.status, 'violated'); + assert.match(result.detail, /was 0/); +}); + +test('a trace whose taps record no metric reports no-data rather than passing', () => { + const result = evaluateInvariant([stopWithMetrics('tapOn', {})], RETRY_INVARIANT); + assert.equal(result.status, 'no-data'); +}); + +// tap-retry-if-no-change is parked (#1300): tapRetries measured 0 then 1 across +// identical runs, so it is flaky rather than divergent. A knownDivergence +// declaration assumes the failure reproduces, so declaring a flaky scenario +// would make the schedule flip red at random. Assert it stays out of the active +// set until it has an inert control — re-adding it without one silently returns +// the coin-flip. +test('the flaky retry scenario is parked, not declared as a divergence', () => { + const retry = DIFFERENTIAL_SCENARIOS.find((s) => s.id === 'tap-retry-if-no-change'); + assert.equal(retry, undefined, 'tap-retry-if-no-change must stay parked until #1300 gives it an inert control'); +}); + +// The invariant itself stays implemented and proven, so the fix PR only has to +// re-add the scenario rather than rebuild the detector. +test('the tapRetries invariant remains implemented for when the scenario returns', () => { + assert.equal(evaluateInvariant([stopWithMetrics('tapOn', { tapRetries: 1 })], RETRY_INVARIANT).status, 'held'); + assert.equal(evaluateInvariant([stopWithMetrics('tapOn', { tapRetries: 0 })], RETRY_INVARIANT).status, 'violated'); +}); + +// Truncation-vs-rounding is at most 1px and cannot be observed on a device, so +// no scenario may claim to guard bug class 1; a unit test pins it instead. +test('no device scenario claims to prove percent truncation (bug class 1)', () => { + const claiming = DIFFERENTIAL_SCENARIOS.filter((s) => s.bugClass === 1); + assert.deepEqual( + claiming.map((s) => s.id), + [], + 'a 1px truncation delta is not app-observable; keep bug class 1 in runtime-port-geometry.test.ts', + ); +}); diff --git a/scripts/maestro-conformance/differential/invariants.ts b/scripts/maestro-conformance/differential/invariants.ts new file mode 100644 index 000000000..ceae1afe4 --- /dev/null +++ b/scripts/maestro-conformance/differential/invariants.ts @@ -0,0 +1,138 @@ +// Engine-side invariants over agent-device's own replay timing trace. +// +// Cross-engine outcome parity (exit codes) only catches a flow that outright +// fails. It cannot see a settle-ordering regression that still passes — which +// matters because layer 3 is the ONLY home for bug class 4 (the 200ms x 10 settle +// loop has no reflectable upstream constant to cross-check in layer 2). +// +// These invariants read the `replay-timing.ndjson` written by the test runtime +// (src/daemon/handlers/session-test-runtime.ts) and assert engine-side facts — +// e.g. a tap must not burn the entire settle budget, which is the signature of a +// stability loop that never latches (a full-budget tap measures ~2093-2117ms +// against a 2000ms budget; a healthy Android tap is ~350ms, iOS ~800-1100ms). +// +// The evaluator is pure and unit-tested against synthetic traces; the device run +// that produces a real trace happens only on the scheduled workflow. +import fs from 'node:fs'; + +export type TraceEvent = { + type: string; + step?: number; + command?: string; + ok?: boolean; + durationMs?: number; + /** Per-step MaestroRuntimeMetrics delta (hierarchyCaptures/screenshotCaptures/tapRetries). */ + resultTiming?: Record; +}; + +export type Invariant = + | { + kind: 'stepDurationBelow'; + /** Maestro command name the step must match (e.g. "tapOn"). */ + command: string; + maxMs: number; + /** Why this bound means the engine behaved correctly. */ + because: string; + } + | { + kind: 'metricAtLeast'; + command: string; + /** MaestroRuntimeMetrics key, recorded per step as a delta. */ + metric: 'tapRetries' | 'hierarchyCaptures' | 'screenshotCaptures'; + min: number; + because: string; + }; + +export type InvariantResult = { + invariant: Invariant; + status: 'held' | 'violated' | 'no-data'; + detail: string; +}; + +/** Parse an ndjson trace, ignoring blank lines and unparseable records. */ +export function readTrace(file: string): TraceEvent[] { + if (!fs.existsSync(file)) return []; + return fs + .readFileSync(file, 'utf8') + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .flatMap((line) => { + try { + return [JSON.parse(line) as TraceEvent]; + } catch { + return []; + } + }); +} + +function completedSteps(events: TraceEvent[], command: string): TraceEvent[] { + return events.filter((event) => event.type === 'replay_action_stop' && event.command === command); +} + +export function evaluateInvariant(events: TraceEvent[], invariant: Invariant): InvariantResult { + const steps = completedSteps(events, invariant.command); + if (steps.length === 0) { + return { + invariant, + status: 'no-data', + detail: `no completed ${invariant.command} steps in the trace`, + }; + } + + if (invariant.kind === 'metricAtLeast') { + const values = steps + .map((step) => step.resultTiming?.[invariant.metric]) + .filter((value): value is number => typeof value === 'number'); + if (values.length === 0) { + return { + invariant, + status: 'no-data', + detail: `no ${invariant.command} step recorded a ${invariant.metric} metric`, + }; + } + // Per-step deltas: the strongest single step is what proves the path ran. + const best = Math.max(...values); + if (best < invariant.min) { + return { + invariant, + status: 'violated', + detail: `highest ${invariant.command} ${invariant.metric} was ${best} (< ${invariant.min}): ${invariant.because}`, + }; + } + return { + invariant, + status: 'held', + detail: `${invariant.command} ${invariant.metric} reached ${best} (>= ${invariant.min})`, + }; + } + + const timed = steps.filter((step) => typeof step.durationMs === 'number'); + if (timed.length === 0) { + return { + invariant, + status: 'no-data', + detail: `no ${invariant.command} step recorded a duration`, + }; + } + const worst = Math.max(...timed.map((step) => step.durationMs ?? 0)); + if (worst >= invariant.maxMs) { + return { + invariant, + status: 'violated', + detail: `slowest ${invariant.command} took ${worst}ms (>= ${invariant.maxMs}ms): ${invariant.because}`, + }; + } + return { + invariant, + status: 'held', + detail: `slowest ${invariant.command} took ${worst}ms (< ${invariant.maxMs}ms)`, + }; +} + +export function evaluateInvariants( + events: TraceEvent[], + invariants: readonly Invariant[], +): InvariantResult[] { + return invariants.map((invariant) => evaluateInvariant(events, invariant)); +} diff --git a/scripts/maestro-conformance/differential/run.test.ts b/scripts/maestro-conformance/differential/run.test.ts new file mode 100644 index 000000000..715fb3548 --- /dev/null +++ b/scripts/maestro-conformance/differential/run.test.ts @@ -0,0 +1,151 @@ +// Device-free self-test for the layer-3 differential registry. Runs in unit CI +// via node --test; the live device comparison itself runs only on the scheduled +// conformance-differential workflow. +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, test } from 'node:test'; +import { + DIFFERENTIAL_APP_ID, + DIFFERENTIAL_SCENARIOS, + type DivergenceSignature, +} from './scenarios.ts'; +import { matchesSignature, parseRunnerArgs, selectScenarios, validateScenarios } from './run.ts'; + +const CONFORMANCE_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +test('every differential scenario references an existing corpus flow with a unique id', () => { + assert.doesNotThrow(() => validateScenarios()); + assert.ok(DIFFERENTIAL_SCENARIOS.length > 0, 'expected at least one differential scenario'); +}); + +test('the settle-loop bug class (4) is covered by a differential scenario', () => { + const settle = DIFFERENTIAL_SCENARIOS.find((scenario) => scenario.bugClass === 4); + assert.ok(settle, 'bug class 4 (settle ordering) has no reflectable constant; it must be a differential scenario'); + assert.equal(settle?.id, 'settle-after-tap'); +}); + +test('--trace-root is accepted so engine-side invariants can be evaluated', () => { + const options = parseRunnerArgs(['--trace-root', '/tmp/artifacts']); + assert.equal(options.traceRoot, '/tmp/artifacts'); +}); + +// The layer-1 corpus exists to be PARSED: its flows name a fictional +// com.example.app and elements that exist on no device. Pointing a device +// scenario at one produces a run that fails before it exercises any runtime +// behavior — which is exactly how the settle detector was silently vacuous. +test('no differential scenario points at the parse-only layer-1 corpus', () => { + const corpusBacked = DIFFERENTIAL_SCENARIOS.filter((s) => s.flow.startsWith('corpus/')); + assert.deepEqual( + corpusBacked.map((s) => s.id), + [], + 'device scenarios must use differential/flows (real fixture app), not the parse corpus', + ); + for (const scenario of DIFFERENTIAL_SCENARIOS) { + assert.ok( + scenario.flow.startsWith('differential/flows/'), + `${scenario.id}: expected a device flow under differential/flows`, + ); + } +}); + +// A declared divergence without an issue behind it is how "temporarily expected" +// becomes permanent without anyone deciding to. Layer 1 requires `unsupported` +// on every we-reject entry for the same reason; enforce the twin here, because +// the lesson of this whole arc is that prose discipline does not survive contact +// with a two-day debugging session. +test('every knownDivergence carries a tracking issue', () => { + const problems: string[] = []; + for (const scenario of DIFFERENTIAL_SCENARIOS) { + const declared = scenario.knownDivergence; + if (!declared) continue; + if (!/^https:\/\/github\.com\/.+\/issues\/\d+$/.test(declared.tracking ?? '')) { + problems.push(`${scenario.id}: knownDivergence.tracking must be a GitHub issue URL`); + } + if (!declared.reason || declared.reason.length < 20) { + problems.push(`${scenario.id}: knownDivergence.reason must explain what it blocks`); + } + } + assert.deepEqual(problems, [], problems.join('\n')); +}); + +// A waiver must cover the ONE failure it was granted for. Without an exact +// signature it is blanket amnesty: while a gap is open the job would also +// swallow upstream regressing or a different invariant breaking — hiding the +// next bug behind the last one. +describe('knownDivergence signature matching', () => { + const sig: DivergenceSignature = { maestro: 'pass', agentDevice: 'fail', invariants: ['no-data'] }; + const engine = (outcome: 'pass' | 'fail') => ({ + engine: 'maestro' as const, + outcome, + exitCode: outcome === 'pass' ? 0 : 1, + }); + const inv = (status: 'held' | 'violated' | 'no-data') => + ({ invariant: { kind: 'stepDurationBelow', command: 'tapOn', maxMs: 1, because: 'x' }, status, detail: '' }) as never; + + test('matches the declared failure exactly', () => { + assert.equal(matchesSignature(sig, engine('pass'), engine('fail'), [inv('no-data')]), true); + }); + + test('upstream also failing is NOT covered by the waiver', () => { + // The #1299 shape is maestro=pass. If Maestro starts failing too, that is a + // different problem and must not ride in green on this declaration. + assert.equal(matchesSignature(sig, engine('fail'), engine('fail'), [inv('no-data')]), false); + }); + + test('our engine unexpectedly passing is NOT covered (declaration is stale)', () => { + assert.equal(matchesSignature(sig, engine('pass'), engine('pass'), [inv('no-data')]), false); + }); + + test('a different invariant outcome is NOT covered', () => { + assert.equal(matchesSignature(sig, engine('pass'), engine('fail'), [inv('violated')]), false); + }); + + test('a new invariant appearing is NOT covered', () => { + assert.equal( + matchesSignature(sig, engine('pass'), engine('fail'), [inv('no-data'), inv('violated')]), + false, + ); + }); + + test('every declaration states its expected signature', () => { + for (const scenario of DIFFERENTIAL_SCENARIOS) { + const declared = scenario.knownDivergence; + if (!declared) continue; + assert.ok(declared.expected, `${scenario.id}: knownDivergence must declare an expected signature`); + assert.ok( + ['pass', 'fail'].includes(declared.expected.maestro) && + ['pass', 'fail'].includes(declared.expected.agentDevice), + `${scenario.id}: signature must state both engines' outcomes`, + ); + // A waiver that expects both engines to pass is not a divergence at all. + assert.ok( + !(declared.expected.maestro === 'pass' && declared.expected.agentDevice === 'pass'), + `${scenario.id}: a signature where both engines pass describes no divergence`, + ); + } + }); +}); + +test('every device flow targets the fixture app the workflow installs', () => { + for (const scenario of DIFFERENTIAL_SCENARIOS) { + const body = fs.readFileSync(path.join(CONFORMANCE_DIR, scenario.flow), 'utf8'); + assert.match( + body, + new RegExp(`^appId:\\s*${DIFFERENTIAL_APP_ID}$`, 'm'), + `${scenario.id} must target ${DIFFERENTIAL_APP_ID}; a flow against any other app cannot run on the CI simulator`, + ); + } +}); + +test('--only selects a single scenario and rejects unknown ids', () => { + assert.equal(selectScenarios('settle-after-tap').length, 1); + assert.throws(() => selectScenarios('does-not-exist'), /No scenario named/); +}); + +test('runner arg parsing honors dry-run and platform', () => { + const options = parseRunnerArgs(['--dry-run', '--platform', 'ios']); + assert.equal(options.dryRun, true); + assert.equal(options.platform, 'ios'); +}); diff --git a/scripts/maestro-conformance/differential/run.ts b/scripts/maestro-conformance/differential/run.ts new file mode 100644 index 000000000..5ec898420 --- /dev/null +++ b/scripts/maestro-conformance/differential/run.ts @@ -0,0 +1,270 @@ +// Layer 3 differential runner. Executes each scenario flow through BOTH real +// Maestro (`maestro test`) and agent-device (`replay`) on a live device and +// compares the observed outcomes. Opt-in: it needs a booted device/simulator, +// the `maestro` CLI on PATH, and an installed target app, so it runs only from +// the scheduled `conformance-differential` workflow or by hand. +// +// node --experimental-strip-types scripts/maestro-conformance/differential/run.ts \ +// --platform ios --out-dir .tmp/conformance-differential +// +// `--dry-run` validates the scenario registry without a device (exercised by +// run.test.ts in unit CI, the same shape as help-conformance-bench). +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + DIFFERENTIAL_SCENARIOS, + type DifferentialScenario, + type DivergenceSignature, +} from './scenarios.ts'; +import { type InvariantResult, evaluateInvariants, readTrace } from './invariants.ts'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const CONFORMANCE_DIR = path.resolve(HERE, '..'); + +export type RunnerOptions = { + platform?: string; + outDir?: string; + /** Artifacts root to search for agent-device's replay-timing.ndjson. */ + traceRoot?: string; + dryRun: boolean; + only?: string; + maestroBin: string; + agentDeviceCli: string; +}; + +export function parseRunnerArgs(argv: readonly string[]): RunnerOptions { + const options: RunnerOptions = { + dryRun: false, + maestroBin: process.env.MAESTRO_BIN ?? 'maestro', + // Mirror the perf harness convention (AGENT_DEVICE_PERF_CLI). + agentDeviceCli: process.env.AGENT_DEVICE_CLI ?? '--experimental-strip-types src/bin.ts', + traceRoot: process.env.AGENT_DEVICE_ARTIFACTS_DIR, + }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--dry-run') options.dryRun = true; + else if (arg === '--platform') options.platform = argv[(i += 1)]; + else if (arg === '--out-dir') options.outDir = argv[(i += 1)]; + else if (arg === '--trace-root') options.traceRoot = argv[(i += 1)]; + else if (arg === '--only') options.only = argv[(i += 1)]; + else throw new Error(`Unknown argument: ${arg}`); + } + return options; +} + +export function selectScenarios(only?: string): DifferentialScenario[] { + if (!only) return DIFFERENTIAL_SCENARIOS; + const selected = DIFFERENTIAL_SCENARIOS.filter((scenario) => scenario.id === only); + if (selected.length === 0) throw new Error(`No scenario named ${only}`); + return selected; +} + +/** Validate the registry without a device: flows exist, ids are unique. */ +export function validateScenarios(): void { + const ids = new Set(); + for (const scenario of DIFFERENTIAL_SCENARIOS) { + if (ids.has(scenario.id)) throw new Error(`Duplicate scenario id: ${scenario.id}`); + ids.add(scenario.id); + const flowPath = path.join(CONFORMANCE_DIR, scenario.flow); + if (!fs.existsSync(flowPath)) throw new Error(`Scenario ${scenario.id} flow not found: ${scenario.flow}`); + } +} + +type EngineResult = { engine: 'maestro' | 'agent-device'; outcome: 'pass' | 'fail'; exitCode: number }; + +function runEngine( + engine: EngineResult['engine'], + command: string, + args: string[], +): EngineResult { + const [bin = '', ...rest] = command.split(' ').filter(Boolean); + const result = spawnSync(bin, [...rest, ...args], { stdio: 'inherit', cwd: process.cwd() }); + const exitCode = result.status ?? 1; + return { engine, outcome: exitCode === 0 ? 'pass' : 'fail', exitCode }; +} + +export type ScenarioReport = { + id: string; + flow: string; + maestro: EngineResult; + agentDevice: EngineResult; + /** Outcome parity across the two engines. */ + outcomeDiverged: boolean; + /** Engine-side invariants over agent-device's own timing trace. */ + invariants: InvariantResult[]; + /** + * ok — behaved as expected. + * failed — an UNDECLARED divergence. The signal this exists for. + * known-divergence — failed exactly as declared; tracked, so the run stays green. + * stale-declaration — passed while still declared divergent: remove the + * declaration (the gap is closed). Fails, so a fix cannot + * land while leaving the oracle blind to a regression. + */ + status: 'ok' | 'failed' | 'known-divergence' | 'stale-declaration'; + tracking?: string; + failed: boolean; +}; + +/** + * Locate the timing trace agent-device wrote for this run. The test runtime + * writes `replay-timing.ndjson` under the run's artifacts directory; we take the + * most recent one so nested attempt-N directories resolve correctly. + */ +function findTimingTrace(root: string | undefined): string | undefined { + if (!root || !fs.existsSync(root)) return undefined; + const found: Array<{ file: string; mtimeMs: number }> = []; + const walk = (dir: string): void => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.name === 'replay-timing.ndjson') { + found.push({ file: full, mtimeMs: fs.statSync(full).mtimeMs }); + } + } + }; + walk(root); + return found.sort((a, b) => b.mtimeMs - a.mtimeMs)[0]?.file; +} + +/** + * Does the observed failure match the one the waiver was granted for, exactly? + * Both engines' outcomes and every declared invariant status must line up; any + * deviation means this is a different failure and must stay red. + */ +export function matchesSignature( + expected: DivergenceSignature, + maestro: EngineResult, + agentDevice: EngineResult, + invariants: InvariantResult[], +): boolean { + if (maestro.outcome !== expected.maestro) return false; + if (agentDevice.outcome !== expected.agentDevice) return false; + const expectedInvariants = expected.invariants ?? []; + if (expectedInvariants.length !== invariants.length) return false; + return expectedInvariants.every((status, index) => invariants[index]?.status === status); +} + +function runScenario(scenario: DifferentialScenario, options: RunnerOptions): ScenarioReport { + const flowPath = path.join(CONFORMANCE_DIR, scenario.flow); + const platformArgs = options.platform ? ['--platform', options.platform] : []; + + const maestro = runEngine('maestro', options.maestroBin, ['test', flowPath, ...platformArgs]); + // `--maestro` is required: without it `test` rejects a .yaml flow outright + // ("test does not support this file type"). Matches scripts/run-test-app-maestro-suite.mjs. + const agentDevice = runEngine('agent-device', `node ${options.agentDeviceCli}`, [ + 'test', + flowPath, + '--maestro', + ...platformArgs, + ]); + + const outcomeDiverged = + maestro.outcome !== scenario.expect || agentDevice.outcome !== scenario.expect; + + // Outcome parity cannot see settle ordering or timing; assert engine-side + // invariants over agent-device's own trace where the scenario declares them. + const trace = findTimingTrace(options.traceRoot); + const invariants = scenario.engineInvariants + ? evaluateInvariants(trace ? readTrace(trace) : [], scenario.engineInvariants) + : []; + const invariantFailed = invariants.some((result) => result.status !== 'held'); + const misbehaved = outcomeDiverged || invariantFailed; + + // A declared divergence is an expected, tracked gap: it keeps the run green so + // the oracle is not blocked on the engine bug it just found. But the waiver + // covers ONE precise failure — matched exactly below — so while the gap is + // open the job still catches anything else (upstream regressing, a different + // invariant breaking). And a declaration that no longer reproduces FAILS, so + // the fix PR has to delete it; that is what turns the differential into the + // acceptance test for its own findings. + const declared = scenario.knownDivergence; + const matchesDeclared = + declared !== undefined && matchesSignature(declared.expected, maestro, agentDevice, invariants); + const status = !declared + ? misbehaved + ? ('failed' as const) + : ('ok' as const) + : matchesDeclared + ? ('known-divergence' as const) + : misbehaved + ? // Diverged, but not the way the waiver describes: not covered. + ('failed' as const) + : ('stale-declaration' as const); + + return { + id: scenario.id, + flow: scenario.flow, + maestro, + agentDevice, + outcomeDiverged, + invariants, + status, + ...(declared ? { tracking: declared.tracking } : {}), + failed: status === 'failed' || status === 'stale-declaration', + }; +} + +function main(argv: readonly string[]): void { + const options = parseRunnerArgs(argv); + validateScenarios(); + const scenarios = selectScenarios(options.only); + + if (options.dryRun) { + for (const scenario of scenarios) { + const invariants = scenario.engineInvariants?.length ?? 0; + const declared = scenario.knownDivergence + ? `\tdeclared-divergence=${scenario.knownDivergence.tracking}` + : ''; + console.log( + `${scenario.id}\t${scenario.flow}\texpect=${scenario.expect}\tinvariants=${invariants}${declared}`, + ); + } + const known = scenarios.filter((scenario) => scenario.knownDivergence).length; + console.log(`\n${scenarios.length} scenario(s) validated, ${known} declared divergence(s).`); + return; + } + + const reports = scenarios.map((scenario) => runScenario(scenario, options)); + if (options.outDir) { + fs.mkdirSync(options.outDir, { recursive: true }); + fs.writeFileSync( + path.join(options.outDir, 'differential-report.json'), + `${JSON.stringify({ platform: options.platform, reports }, null, 2)}\n`, + ); + } + + for (const report of reports) { + console.log( + `${report.status.padEnd(17)} ${report.id} maestro=${report.maestro.outcome} agent-device=${report.agentDevice.outcome}`, + ); + for (const result of report.invariants) { + console.log(` invariant ${result.status}: ${result.detail}`); + } + if (report.status === 'known-divergence') { + console.log(` declared divergence, tracked: ${report.tracking}`); + } + if (report.status === 'stale-declaration') { + console.log( + ` passed while declared divergent — remove knownDivergence (${report.tracking}) so this stays enforced`, + ); + } + } + + // Keep declared gaps visible: a green run must still say what it is not proving. + const known = reports.filter((report) => report.status === 'known-divergence'); + if (known.length > 0) { + console.log(`\n${known.length} declared divergence(s), not enforced: ${known.map((r) => r.id).join(', ')}`); + } + + const failed = reports.filter((report) => report.failed); + if (failed.length > 0) { + console.error(`\n${failed.length} scenario(s) failed: ${failed.map((r) => r.id).join(', ')}`); + process.exitCode = 1; + } +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(process.argv.slice(2)); +} diff --git a/scripts/maestro-conformance/differential/scenarios.ts b/scripts/maestro-conformance/differential/scenarios.ts new file mode 100644 index 000000000..57f5afa81 --- /dev/null +++ b/scripts/maestro-conformance/differential/scenarios.ts @@ -0,0 +1,163 @@ +// Layer 3 — app-observable differential scenarios. +// +// A small set of flows run through BOTH real Maestro and agent-device on a live +// device. Opt-in/dispatch — never part of per-PR unit CI. +// +// These flows live in ./flows and target the real fixture app +// (examples/test-app, `com.callstack.agentdevicelab`), which the workflow builds +// and installs. They are deliberately NOT the layer-1 corpus: those flows exist +// only to be PARSED — they name a fictional `com.example.app` and elements that +// exist nowhere — so pointing a device run at them would fail before exercising +// any runtime behavior. +// +// Read the field names literally. Cross-engine comparison is OUTCOME parity (does +// the flow pass on both engines), which only catches a divergence severe enough +// to fail the flow. Anything finer — settle latching, retap counts, truncated +// pixel coordinates — is not visible to outcome parity, so where we can assert it +// we do it engine-side via `engineInvariants` over agent-device's own replay +// timing trace. Scenarios without invariants prove outcome parity ONLY; do not +// read more into them than that. +import { MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS } from '../../../src/compat/maestro/compatibility-policy.ts'; +import type { Invariant } from './invariants.ts'; + +/** Bundle id of the fixture app the workflow installs before running scenarios. */ +export const DIFFERENTIAL_APP_ID = 'com.callstack.agentdevicelab'; + +export type DifferentialOutcome = 'pass' | 'fail'; + +/** + * A divergence this scenario is currently EXPECTED to exhibit. + * + * This is the layer-3 twin of layer 1's FLOW_DIVERGENCES, and it exists for the + * same reason: the oracle's contract is that every divergence is a decision on + * the record, never a silent one. When the differential catches a real engine + * bug, the instrument must not be blocked on repairing what it just measured — + * declare it, file it, and let the scheduled run stay green on the known gap + * while still failing on anything undeclared. + * + * `tracking` is REQUIRED and enforced by run.test.ts. A declared divergence with + * no issue behind it is exactly how "temporarily expected" becomes permanent + * without anyone deciding to make it so. + */ +/** + * The EXACT failure a declaration covers. + * + * Without this a declaration is blanket amnesty: any failure at all would turn + * the scenario green, so while a gap is open the job would also swallow an + * unrelated regression — upstream Maestro starting to fail, or a different + * invariant breaking. A waiver must cover the one failure it was granted for and + * nothing else, so the signature is matched exactly and any deviation is red. + */ +export type DivergenceSignature = { + /** Outcome each engine is expected to produce while the gap is open. */ + maestro: DifferentialOutcome; + agentDevice: DifferentialOutcome; + /** Expected status of each declared engine invariant, in declaration order. */ + invariants?: Array<'held' | 'violated' | 'no-data'>; +}; + +export type KnownDivergence = { + /** Why this scenario currently fails, and what it blocks. */ + reason: string; + /** Issue tracking the fix. Required — see above. */ + tracking: string; + /** The precise failure this waiver covers. Anything else stays red. */ + expected: DivergenceSignature; +}; + +export type DifferentialScenario = { + id: string; + /** The #1217 bug class this scenario guards, when applicable. */ + bugClass?: 1 | 2 | 3 | 4; + /** Corpus flow, relative to scripts/maestro-conformance/. */ + flow: string; + /** Exactly what running both engines and comparing outcomes establishes. */ + comparesAcrossEngines: string; + /** Expected outcome from BOTH engines when parity holds. */ + expect: DifferentialOutcome; + /** Machine-checkable assertions over agent-device's own timing trace. */ + engineInvariants?: Invariant[]; + /** What a divergence would indicate. */ + divergenceMeans: string; + /** + * Declared, tracked failure. The run stays green while this holds — but it + * FAILS if the scenario starts passing, so the fix PR must remove the + * declaration and the oracle enforces that the gap stays closed. + */ + knownDivergence?: KnownDivergence; +}; + +export const DIFFERENTIAL_SCENARIOS: DifferentialScenario[] = [ + { + id: 'settle-after-tap', + bugClass: 4, + flow: 'differential/flows/settle-after-tap.yaml', + comparesAcrossEngines: 'The tap succeeds on both engines.', + expect: 'pass', + // Outcome parity cannot see settle ordering: a tap that burns the whole + // budget still passes. This invariant is the actual bug-class-4 detector. + engineInvariants: [ + { + kind: 'stepDurationBelow', + command: 'tapOn', + maxMs: MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS, + because: + 'a tap consuming the entire settle budget means the stability loop never latched — the signature of a sleep-before-capture ordering regression', + }, + ], + divergenceMeans: + 'agent-device settled in a different order than upstream (sleep-before vs sleep-after capture) or never latches within the shared budget.', + knownDivergence: { + reason: + 'Caught by this differential on its first working run: our scrollUntilVisible times out finding home-open-form where Maestro 2.5.1 scrolls to it and passes. The flow cannot reach its tapOn step, so bug class 4 has no working device detector until that engine bug is fixed. Declared rather than chased inline — the instrument does not block on repairing what it just measured.', + tracking: 'https://github.com/callstack/agent-device/issues/1299', + // Exactly what runs 29504440599 and 29510020718 observed. If upstream + // starts failing too, or the flow fails for a different reason, the + // signature stops matching and the job goes red — a waiver for THIS bug + // must not silently cover the next one. + expected: { + maestro: 'pass', + agentDevice: 'fail', + // The tap step is never reached, so the settle invariant has no data. + invariants: ['no-data'], + }, + }, + }, + { + id: 'percent-swipe', + // Deliberately NOT tagged bugClass 1. Truncation-vs-rounding is at most a + // one-pixel difference: no app-observable outcome on a real device can + // distinguish it, so claiming this scenario guards bug class 1 would be a + // lie the pass/pass result cannot back up. That half is pinned exactly by a + // pure unit test of the conversion + // (src/compat/maestro/__tests__/runtime-port-geometry.test.ts), and the + // parse-level half by layer 1 (bug-classes/percent-decimal-swipe). What this + // scenario adds is only that percentage swipes work end-to-end on both. + flow: 'differential/flows/percent-swipe.yaml', + comparesAcrossEngines: + 'A percentage-endpoint swipe completes on both engines. It does NOT compare resolved pixel endpoints — no metric records them and a 1px delta is not app-observable.', + expect: 'pass', + divergenceMeans: 'The swipe behaves differently enough on one engine to fail the flow.', + }, + // PARKED: tap-retry-if-no-change (flow kept at differential/flows/, invariant + // kept implemented and unit-tested) — https://github.com/callstack/agent-device/issues/1300 + // + // It is NOT declared as a knownDivergence, deliberately. tapRetries measured 0 + // in run 29504440599 and 1 in run 29510020718 with no change to the flow or + // commit: the tap sometimes holds the hierarchy signature still and sometimes + // does not, because the fixture home screen has live content. A declaration + // assumes the divergence REPRODUCES; a flaky one would flip between + // known-divergence (green) and stale-declaration (red) at random, which is + // worse than no scenario — a coin-flip job teaches people to ignore the + // differential. So retryIfNoChange has no device coverage until the fixture + // offers an inert control, and that absence is tracked rather than disguised + // as a green run. + { + id: 'optional-warned-not-failed', + flow: 'differential/flows/optional-warned-not-failed.yaml', + comparesAcrossEngines: + 'An optional assertion on an element that never exists is downgraded to a warning and the flow still completes on both engines — a failed-instead-of-warned classification flips the exit code, so outcome parity does prove this one.', + expect: 'pass', + divergenceMeans: 'agent-device failed an optional command upstream would have warned on.', + }, +]; diff --git a/scripts/maestro-conformance/expected-divergence.ts b/scripts/maestro-conformance/expected-divergence.ts new file mode 100644 index 000000000..9127d3dce --- /dev/null +++ b/scripts/maestro-conformance/expected-divergence.ts @@ -0,0 +1,178 @@ +// Declared, intentional divergences between agent-device and upstream Maestro. +// The verifier fails on ANY undeclared divergence, so every entry here is a +// decision on the record rather than silent drift. + +export type FlowDivergence = { + /** Expected non-identical classification for this corpus flow. */ + classification: 'we-reject' | 'mismatch'; + /** Why the divergence is intentional. */ + reason: string; + /** Upstream commands/options our engine deliberately does not support. */ + unsupported?: string[]; + /** Tracking issue, when the divergence is a backlog item rather than a decision. */ + tracking?: string; +}; + +// Keyed by corpus flow id. Every flow the verifier classifies as `we-reject` or +// `mismatch` must appear here; anything undeclared fails the suite. This is the +// mechanical parity backlog — the list the old hand-typed fixture could not +// produce. `tracking` points at the Maestro compat tracker for option-level gaps. +const COMPAT_TRACKER = 'https://github.com/callstack/agent-device/issues/558'; + +export const FLOW_DIVERGENCES: Record = { + // --- Unsupported commands (support-matrix decisions) --- + 'upstream/045_clear_keychain': { + classification: 'we-reject', + reason: 'Standalone clearKeychain is outside the supported subset.', + unsupported: ['clearKeychain'], + }, + 'upstream/051_set_location': { + classification: 'we-reject', + reason: 'setLocation is outside the supported subset.', + unsupported: ['setLocation'], + }, + 'upstream/062_copy_paste_text': { + classification: 'we-reject', + reason: 'Clipboard commands are unsupported; pasteText is de-advertised (takes clipboard, not inline text).', + unsupported: ['copyTextFrom', 'pasteText'], + }, + 'upstream/067_assertTrue_pass': { + classification: 'we-reject', + reason: 'assertTrue is outside the supported subset.', + unsupported: ['assertTrue'], + tracking: COMPAT_TRACKER, + }, + 'upstream/090_travel': { + classification: 'we-reject', + reason: 'travel is outside the supported subset.', + unsupported: ['travel'], + }, + 'upstream/116_kill_app': { + classification: 'we-reject', + reason: 'Standalone killApp is outside the supported subset.', + unsupported: ['killApp'], + }, + 'upstream/131_setPermissions': { + classification: 'we-reject', + reason: 'Standalone setPermissions is outside the supported subset (launchApp permissions are supported).', + unsupported: ['setPermissions'], + }, + 'upstream/053_repeat_times': { + classification: 'we-reject', + reason: 'repeat is supported, but the flow also uses evalScript and a ${output.list.length} times expression.', + unsupported: ['evalScript'], + tracking: COMPAT_TRACKER, + }, + // --- Deliberately stricter than upstream --- + 'invalid/duplicate-keys': { + classification: 'we-reject', + reason: + 'Upstream (SnakeYAML) silently accepts duplicate mapping keys and keeps the last one; our parser rejects them. Being stricter than upstream on a near-certain authoring mistake is intentional — surfacing it beats silently dropping a selector the author wrote.', + unsupported: ['duplicate YAML mapping keys'], + }, + // --- Unsupported options on supported commands (parity gaps) --- + 'upstream/032_element_index': { + classification: 'we-reject', + reason: 'A literal tapOn.index is supported; the flow also uses a ${0 + 1} JS-expression index.', + unsupported: ['tapOn.index (expression)'], + tracking: COMPAT_TRACKER, + }, + 'upstream/034_press_key': { + classification: 'we-reject', + reason: 'pressKey supports back/enter/home/return; the flow exercises ~30 Android/TV keycodes.', + unsupported: ['pressKey (extended keycodes)'], + tracking: COMPAT_TRACKER, + }, + 'upstream/042_extended_wait': { + classification: 'we-reject', + reason: 'A literal extendedWaitUntil.timeout is supported; the flow interpolates ${TIMEOUT} from a flow env block (unresolved ${} is fail-loud).', + unsupported: ['extendedWaitUntil.timeout (interpolation)'], + tracking: COMPAT_TRACKER, + }, + 'upstream/076_optional_assertion': { + classification: 'we-reject', + reason: 'optional is supported on tapOn/assertion targets; the flow marks scrollUntilVisible/extendedWaitUntil optional and uses assertTrue.', + unsupported: ['optional (scrollUntilVisible/extendedWaitUntil)', 'assertTrue'], + tracking: COMPAT_TRACKER, + }, + 'upstream/079_scroll_until_visible': { + classification: 'we-reject', + reason: 'scrollUntilVisible is supported; the flow sets the unsupported speed and visibilityPercentage options.', + unsupported: ['scrollUntilVisible.speed', 'scrollUntilVisible.visibilityPercentage'], + tracking: COMPAT_TRACKER, + }, + 'upstream/101_doubleTapOn': { + classification: 'we-reject', + reason: 'doubleTapOn is supported; the flow sets the unsupported retryTapIfNoChange option on it.', + unsupported: ['doubleTapOn.retryTapIfNoChange'], + tracking: COMPAT_TRACKER, + }, + 'upstream/114_child_of_selector': { + classification: 'we-reject', + reason: 'childOf is supported on tapOn but not on assertVisible/assertNotVisible (structural selector).', + unsupported: ['assertVisible.childOf', 'assertNotVisible.childOf'], + tracking: COMPAT_TRACKER, + }, + 'upstream/119_retry_commands': { + classification: 'we-reject', + reason: 'retry is supported; the flow uses the unsupported per-command waitToSettleTimeoutMs option on a nested tapOn.', + unsupported: ['tapOn.waitToSettleTimeoutMs'], + tracking: COMPAT_TRACKER, + }, +}; + +// Layer-2 semantic vectors that describe an upstream constant we intentionally +// do NOT mirror (they document a deviation rather than a value to match). +export const LAYER2_REFERENCE_ONLY = new Set([ + // Upstream iOS taps wait up to 3s for the screen to become static BEFORE + // tapping (IOSDriver.SCREEN_SETTLE_TIMEOUT_MS). agent-device deliberately omits + // this pre-tap gate and masks entrance-animation flake with quantized-signature + // retryIfNoChange instead. See ADR-0015. + 'iosScreenSettleTimeoutMs', +]); + +// Supported commands that no corpus flow exercises and are therefore verified by +// other means (or explicitly deferred). Listed so coverage stays honest. +export const UNVERIFIED_COMMANDS = new Set([]); + +// Behavioral deviations that are decisions, not parser-level mismatches. These +// are not tied to a single corpus flow; they are recorded so the support matrix +// and ADR-0015 stay in sync with the oracle. +export type DocumentedDeviation = { + id: string; + area: string; + description: string; +}; + +export const DOCUMENTED_DEVIATIONS: DocumentedDeviation[] = [ + { + id: 'animation-wait-mechanism', + area: 'waitForAnimationToEnd', + description: + 'Upstream waitForAnimationToEnd diffs full-screen screenshots; agent-device diffs a typed-hierarchy signature and only falls back to the screenshot metric. The 0.005 threshold and 15000ms timeout constants match (layer-2 crosscheck); the stabilization mechanism differs by design.', + }, + { + id: 'horizontal-swipe-presets', + area: 'swipe', + description: + 'Vertical directional swipes use the upstream 0.1/0.9 edge fractions; horizontal directional swipes route through the shared in-page gesture planner to avoid triggering the OS interactive-back edge gesture.', + }, + { + id: 'condition-poll-as-stabilization', + area: 'settle', + description: + 'Upstream settle busy-polls after gestures; agent-device folds condition polling into the same settle protocol (execute drains a stability barrier, observe condition-polls).', + }, + { + id: 'ambiguity-strictness-under-optional', + area: 'optional', + description: + 'Upstream target resolution selects the first match and never raises ambiguity. agent-device keeps first-match parity for Maestro selectors, but AMBIGUOUS_MATCH surfacing from nested shared public commands is non-suppressible under optional (policy-layer strictness, not an upstream-parity vector).', + }, + { + id: 'ios-pre-tap-static-gate-omitted', + area: 'tap', + description: + 'agent-device omits the upstream iOS 3s pre-tap static-screen gate (IOSDriver.SCREEN_SETTLE_TIMEOUT_MS); see LAYER2_REFERENCE_ONLY.', + }, +]; diff --git a/scripts/maestro-conformance/fixture-seal.mjs b/scripts/maestro-conformance/fixture-seal.mjs new file mode 100644 index 000000000..06d531925 --- /dev/null +++ b/scripts/maestro-conformance/fixture-seal.mjs @@ -0,0 +1,29 @@ +// Content seal shared by the generator (regenerate.mjs) and the deterministic +// verifier (verify.ts). +// +// Normal CI cannot re-derive the fixtures — that needs Java, and "no Java in +// normal CI" is a design constraint. So per-PR verification recomputes this seal +// over the generated content: any hand edit to a captured command or constant +// changes the content and fails the check. That makes casual or accidental +// hand-editing impossible rather than merely discouraged. +// +// The seal is tamper-EVIDENT, not tamper-proof: someone could edit the content +// and recompute the hash. The scheduled `conformance-regenerate` job closes that +// hole for real by re-running the JVM harness against the pinned artifacts and +// failing on any byte difference — forgery cannot survive an actual re-derivation. +import { createHash } from 'node:crypto'; + +/** + * Hash a fixture's generated content. The argument must NOT contain the seal + * itself; `regenerate.mjs` appends `contentHash` last, so stripping that key + * reproduces the exact object (and key order) that was hashed at write time. + */ +export function fixtureContentHash(fixtureWithoutSeal) { + return createHash('sha256').update(JSON.stringify(fixtureWithoutSeal)).digest('hex'); +} + +/** Recompute a parsed fixture's seal. Returns the expected and recorded hashes. */ +export function checkFixtureSeal(parsedFixture) { + const { contentHash, ...content } = parsedFixture; + return { expected: fixtureContentHash(content), actual: contentHash }; +} diff --git a/scripts/maestro-conformance/fixtures/layer1-parser.json b/scripts/maestro-conformance/fixtures/layer1-parser.json new file mode 100644 index 000000000..85c12a85c --- /dev/null +++ b/scripts/maestro-conformance/fixtures/layer1-parser.json @@ -0,0 +1,3923 @@ +{ + "schemaVersion": 2, + "generatedBy": "scripts/maestro-conformance/regenerate.mjs", + "upstream": { + "project": "mobile-dev-inc/Maestro", + "version": "2.5.1", + "tag": "v2.5.1", + "commit": "a4c7c95f5ba1884858f7e35efa6b8e0165db9448", + "artifacts": [ + { + "coordinate": "dev.mobile:maestro-orchestra:2.5.1", + "role": "Layer 1 YAML parser (YamlCommandReader) and Layer 2 retry/erase constants (Orchestra).", + "sha256": "bbd8e5e35e3c706c6d4760bb1026162474b171155a5fa76f6de1360ab3705468" + }, + { + "coordinate": "dev.mobile:maestro-orchestra-models:2.5.1", + "role": "Command model classes and parser-observed defaults (Layer 2 model defaults).", + "sha256": "0c147f7dff3704e1e04cb728339f2637a6086c234d7984c1434536680f6749ee" + }, + { + "coordinate": "dev.mobile:maestro-client:2.5.1", + "role": "Layer 2 driver constants (Maestro.SCREENSHOT_DIFF_THRESHOLD/ANIMATION_TIMEOUT_MS, IOSDriver.SCREEN_SETTLE_TIMEOUT_MS).", + "sha256": "a6b8c63c0858e8e256752f617fd27b35b0be48e78d367f9cbdffa735a6fcc991" + } + ] + }, + "layer": 1, + "description": "Upstream Maestro parser capture. Generated by the JVM harness; do not hand-edit.", + "flows": [ + { + "id": "upstream/001_assert_visible_by_id", + "file": "upstream/001_assert_visible_by_id.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": { + "textRegex": null, + "idRegex": "element_id", + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "notVisible": null, + "scriptCondition": null, + "label": null + }, + "timeout": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/002_assert_visible_by_text", + "file": "upstream/002_assert_visible_by_text.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": { + "textRegex": "Element Text", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "notVisible": null, + "scriptCondition": null, + "label": null + }, + "timeout": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/008_tap_on_element", + "file": "upstream/008_tap_on_element.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": ".*button.*", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/009_skip_optional_elements", + "file": "upstream/009_skip_optional_elements.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": "Optional Element", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": true, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": true + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": { + "textRegex": "Non Optional", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "notVisible": null, + "scriptCondition": null, + "label": null + }, + "timeout": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/010_scroll", + "file": "upstream/010_scroll.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "ScrollCommand", + "fields": { + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/011_back_press", + "file": "upstream/011_back_press.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "BackPressCommand", + "fields": { + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/012_input_text", + "file": "upstream/012_input_text.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "InputTextCommand", + "fields": { + "text": "Hello World", + "label": null, + "optional": false + } + }, + { + "type": "InputTextCommand", + "fields": { + "text": "user@example.com", + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/013_launch_app", + "file": "upstream/013_launch_app.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "LaunchAppCommand", + "fields": { + "appId": "com.example.app", + "clearState": null, + "clearKeychain": null, + "stopApp": null, + "permissions": null, + "launchArguments": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/014_tap_on_point", + "file": "upstream/014_tap_on_point.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnPointV2Command", + "fields": { + "point": "100,200", + "retryIfNoChange": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/017_swipe", + "file": "upstream/017_swipe.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "SwipeCommand", + "fields": { + "direction": null, + "startPoint": { + "x": 100, + "y": 500 + }, + "endPoint": { + "x": 100, + "y": 200 + }, + "elementSelector": null, + "startRelative": null, + "endRelative": null, + "duration": 3000, + "waitToSettleTimeoutMs": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/021_launch_app_with_clear_state", + "file": "upstream/021_launch_app_with_clear_state.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "LaunchAppCommand", + "fields": { + "appId": "com.example.app", + "clearState": true, + "clearKeychain": null, + "stopApp": null, + "permissions": null, + "launchArguments": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/026_assert_not_visible", + "file": "upstream/026_assert_not_visible.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": null, + "notVisible": { + "textRegex": null, + "idRegex": "element_id", + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "scriptCondition": null, + "label": null + }, + "timeout": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/027_open_link", + "file": "upstream/027_open_link.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "OpenLinkCommand", + "fields": { + "link": "https://example.com", + "autoVerify": false, + "browser": false, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/029_long_press_on_element", + "file": "upstream/029_long_press_on_element.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": ".*button.*", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": true, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/032_element_index", + "file": "upstream/032_element_index.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": "Item.*", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": "0", + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": "Item.*", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": "${0 + 1}", + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/034_press_key", + "file": "upstream/034_press_key.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "ENTER", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "BACKSPACE", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "HOME", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "BACK", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "VOLUME_UP", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "VOLUME_DOWN", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "LOCK", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_UP", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_DOWN", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_LEFT", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_RIGHT", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_CENTER", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_PLAY_PAUSE", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_STOP", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_NEXT", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_PREVIOUS", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_REWIND", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_FAST_FORWARD", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "POWER", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "TAB", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_SYSTEM_NAVIGATION_UP", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_SYSTEM_NAVIGATION_DOWN", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_BUTTON_A", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_BUTTON_B", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "REMOTE_MENU", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "TV_INPUT", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "TV_INPUT_HDMI_1", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "TV_INPUT_HDMI_2", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "TV_INPUT_HDMI_3", + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/036_erase_text", + "file": "upstream/036_erase_text.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "InputTextCommand", + "fields": { + "text": "Hello World", + "label": null, + "optional": false + } + }, + { + "type": "EraseTextCommand", + "fields": { + "charactersToErase": 6, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/039_hide_keyboard", + "file": "upstream/039_hide_keyboard.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "HideKeyboardCommand", + "fields": { + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/041_take_screenshot", + "file": "upstream/041_take_screenshot.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TakeScreenshotCommand", + "fields": { + "path": "${MAESTRO_FILENAME}_with_filename", + "cropOn": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/042_extended_wait", + "file": "upstream/042_extended_wait.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "DefineVariablesCommand", + "fields": { + "env": { + "TIMEOUT": "1000" + }, + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": { + "textRegex": "Item", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "notVisible": null, + "scriptCondition": null, + "label": null + }, + "timeout": "${TIMEOUT}", + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": null, + "notVisible": { + "textRegex": "Another item", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "scriptCondition": null, + "label": null + }, + "timeout": "1000", + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": { + "textRegex": "Item", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "notVisible": null, + "scriptCondition": null, + "label": null + }, + "timeout": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/043_stop_app", + "file": "upstream/043_stop_app.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "StopAppCommand", + "fields": { + "appId": "com.example.app", + "label": null, + "optional": false + } + }, + { + "type": "StopAppCommand", + "fields": { + "appId": "another.app", + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/045_clear_keychain", + "file": "upstream/045_clear_keychain.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "ClearKeychainCommand", + "fields": { + "label": null, + "optional": false + } + }, + { + "type": "LaunchAppCommand", + "fields": { + "appId": "com.example.app", + "clearState": null, + "clearKeychain": true, + "stopApp": null, + "permissions": null, + "launchArguments": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/051_set_location", + "file": "upstream/051_set_location.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "LaunchAppCommand", + "fields": { + "appId": "com.example.app", + "clearState": null, + "clearKeychain": null, + "stopApp": null, + "permissions": null, + "launchArguments": null, + "label": null, + "optional": false + } + }, + { + "type": "SetLocationCommand", + "fields": { + "latitude": "12.5266", + "longitude": "78.2150", + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/053_repeat_times", + "file": "upstream/053_repeat_times.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.other.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "RepeatCommand", + "fields": { + "times": "3", + "condition": null, + "commands": [ + { + "tapOnElement": { + "selector": { + "textRegex": "Button", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + }, + "tapOnPoint": null, + "tapOnPointV2Command": null, + "scrollCommand": null, + "swipeCommand": null, + "backPressCommand": null, + "assertCommand": null, + "assertConditionCommand": null, + "assertScreenshotCommand": null, + "assertNoDefectsWithAICommand": null, + "assertWithAICommand": null, + "extractTextWithAICommand": null, + "inputTextCommand": null, + "inputRandomTextCommand": null, + "launchAppCommand": null, + "setPermissionsCommand": null, + "applyConfigurationCommand": null, + "openLinkCommand": null, + "pressKeyCommand": null, + "eraseTextCommand": null, + "hideKeyboardCommand": null, + "takeScreenshotCommand": null, + "stopAppCommand": null, + "killAppCommand": null, + "clearStateCommand": null, + "clearKeychainCommand": null, + "runFlowCommand": null, + "setLocationCommand": null, + "setOrientationCommand": null, + "repeatCommand": null, + "copyTextCommand": null, + "setClipboardCommand": null, + "pasteTextCommand": null, + "defineVariablesCommand": null, + "runScriptCommand": null, + "waitForAnimationToEndCommand": null, + "evalScriptCommand": null, + "scrollUntilVisible": null, + "travelCommand": null, + "startRecordingCommand": null, + "stopRecordingCommand": null, + "addMediaCommand": null, + "setAirplaneModeCommand": null, + "toggleAirplaneModeCommand": null, + "retryCommand": null + } + ], + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": { + "textRegex": "3", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "notVisible": null, + "scriptCondition": null, + "label": null + }, + "timeout": null, + "label": null, + "optional": false + } + }, + { + "type": "EvalScriptCommand", + "fields": { + "scriptString": "${output.list = [1, 2, 3]}", + "label": null, + "optional": false + } + }, + { + "type": "RepeatCommand", + "fields": { + "times": "${output.list.length}", + "condition": null, + "commands": [ + { + "tapOnElement": { + "selector": { + "textRegex": "Button", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + }, + "tapOnPoint": null, + "tapOnPointV2Command": null, + "scrollCommand": null, + "swipeCommand": null, + "backPressCommand": null, + "assertCommand": null, + "assertConditionCommand": null, + "assertScreenshotCommand": null, + "assertNoDefectsWithAICommand": null, + "assertWithAICommand": null, + "extractTextWithAICommand": null, + "inputTextCommand": null, + "inputRandomTextCommand": null, + "launchAppCommand": null, + "setPermissionsCommand": null, + "applyConfigurationCommand": null, + "openLinkCommand": null, + "pressKeyCommand": null, + "eraseTextCommand": null, + "hideKeyboardCommand": null, + "takeScreenshotCommand": null, + "stopAppCommand": null, + "killAppCommand": null, + "clearStateCommand": null, + "clearKeychainCommand": null, + "runFlowCommand": null, + "setLocationCommand": null, + "setOrientationCommand": null, + "repeatCommand": null, + "copyTextCommand": null, + "setClipboardCommand": null, + "pasteTextCommand": null, + "defineVariablesCommand": null, + "runScriptCommand": null, + "waitForAnimationToEndCommand": null, + "evalScriptCommand": null, + "scrollUntilVisible": null, + "travelCommand": null, + "startRecordingCommand": null, + "stopRecordingCommand": null, + "addMediaCommand": null, + "setAirplaneModeCommand": null, + "toggleAirplaneModeCommand": null, + "retryCommand": null + } + ], + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": { + "textRegex": "6", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "notVisible": null, + "scriptCondition": null, + "label": null + }, + "timeout": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/059_directional_swipe_command", + "file": "upstream/059_directional_swipe_command.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "SwipeCommand", + "fields": { + "direction": "RIGHT", + "startPoint": null, + "endPoint": null, + "elementSelector": null, + "startRelative": null, + "endRelative": null, + "duration": 500, + "waitToSettleTimeoutMs": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/061_launchApp_withoutStopping", + "file": "upstream/061_launchApp_withoutStopping.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "LaunchAppCommand", + "fields": { + "appId": "com.example.app", + "clearState": null, + "clearKeychain": null, + "stopApp": false, + "permissions": null, + "launchArguments": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/062_copy_paste_text", + "file": "upstream/062_copy_paste_text.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "CopyTextFromCommand", + "fields": { + "selector": { + "textRegex": null, + "idRegex": "myId", + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "label": null, + "optional": false + } + }, + { + "type": "PasteTextCommand", + "fields": { + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/067_assertTrue_pass", + "file": "upstream/067_assertTrue_pass.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": null, + "notVisible": null, + "scriptCondition": "${1+1}", + "label": null + }, + "timeout": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/069_wait_for_animation_to_end", + "file": "upstream/069_wait_for_animation_to_end.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "WaitForAnimationToEndCommand", + "fields": { + "timeout": "500", + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/074_directional_swipe_element", + "file": "upstream/074_directional_swipe_element.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "SwipeCommand", + "fields": { + "direction": "RIGHT", + "startPoint": null, + "endPoint": null, + "elementSelector": { + "textRegex": "swiping element", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "startRelative": null, + "endRelative": null, + "duration": 400, + "waitToSettleTimeoutMs": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/076_optional_assertion", + "file": "upstream/076_optional_assertion.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "ScrollUntilVisibleCommand", + "fields": { + "selector": { + "textRegex": null, + "idRegex": "not_found", + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "direction": "DOWN", + "scrollDuration": "40", + "visibilityPercentage": 100, + "timeout": "1", + "waitToSettleTimeoutMs": null, + "centerElement": false, + "originalSpeedValue": "40", + "label": null, + "optional": true, + "visibilityPercentageNormalized": 1 + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": null, + "notVisible": null, + "scriptCondition": "false", + "label": null + }, + "timeout": null, + "label": null, + "optional": true + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": { + "textRegex": null, + "idRegex": "not_found", + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "notVisible": null, + "scriptCondition": null, + "label": null + }, + "timeout": "1", + "label": null, + "optional": true + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": { + "textRegex": "Button", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": true, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "notVisible": null, + "scriptCondition": null, + "label": null + }, + "timeout": null, + "label": null, + "optional": true + } + } + ] + }, + { + "id": "upstream/078_swipe_relative", + "file": "upstream/078_swipe_relative.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "SwipeCommand", + "fields": { + "direction": null, + "startPoint": null, + "endPoint": null, + "elementSelector": null, + "startRelative": "50%,30%", + "endRelative": "50%,60%", + "duration": 3000, + "waitToSettleTimeoutMs": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/079_scroll_until_visible", + "file": "upstream/079_scroll_until_visible.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "ScrollUntilVisibleCommand", + "fields": { + "selector": { + "textRegex": "Test", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "direction": "DOWN", + "scrollDuration": "100", + "visibilityPercentage": 100, + "timeout": "10", + "waitToSettleTimeoutMs": null, + "centerElement": false, + "originalSpeedValue": "100", + "label": null, + "optional": false, + "visibilityPercentageNormalized": 1 + } + } + ] + }, + { + "id": "upstream/090_travel", + "file": "upstream/090_travel.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TravelCommand", + "fields": { + "points": [ + { + "latitude": "0.0", + "longitude": "0.0" + }, + { + "latitude": "0.1", + "longitude": "0.0" + }, + { + "latitude": "0.1", + "longitude": "0.1" + }, + { + "latitude": "0.0", + "longitude": "0.1" + } + ], + "speedMPS": 7900, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/094_runFlow_inline", + "file": "upstream/094_runFlow_inline.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "RunFlowCommand", + "fields": { + "commands": [ + { + "tapOnElement": null, + "tapOnPoint": null, + "tapOnPointV2Command": null, + "scrollCommand": null, + "swipeCommand": null, + "backPressCommand": null, + "assertCommand": null, + "assertConditionCommand": null, + "assertScreenshotCommand": null, + "assertNoDefectsWithAICommand": null, + "assertWithAICommand": null, + "extractTextWithAICommand": null, + "inputTextCommand": null, + "inputRandomTextCommand": null, + "launchAppCommand": null, + "setPermissionsCommand": null, + "applyConfigurationCommand": null, + "openLinkCommand": null, + "pressKeyCommand": null, + "eraseTextCommand": null, + "hideKeyboardCommand": null, + "takeScreenshotCommand": null, + "stopAppCommand": null, + "killAppCommand": null, + "clearStateCommand": null, + "clearKeychainCommand": null, + "runFlowCommand": null, + "setLocationCommand": null, + "setOrientationCommand": null, + "repeatCommand": null, + "copyTextCommand": null, + "setClipboardCommand": null, + "pasteTextCommand": null, + "defineVariablesCommand": { + "env": { + "INNER_ENV": "Inner Parameter" + }, + "label": null, + "optional": false + }, + "runScriptCommand": null, + "waitForAnimationToEndCommand": null, + "evalScriptCommand": null, + "scrollUntilVisible": null, + "travelCommand": null, + "startRecordingCommand": null, + "stopRecordingCommand": null, + "addMediaCommand": null, + "setAirplaneModeCommand": null, + "toggleAirplaneModeCommand": null, + "retryCommand": null + }, + { + "tapOnElement": null, + "tapOnPoint": null, + "tapOnPointV2Command": null, + "scrollCommand": null, + "swipeCommand": null, + "backPressCommand": null, + "assertCommand": null, + "assertConditionCommand": null, + "assertScreenshotCommand": null, + "assertNoDefectsWithAICommand": null, + "assertWithAICommand": null, + "extractTextWithAICommand": null, + "inputTextCommand": { + "text": "${INNER_ENV}", + "label": null, + "optional": false + }, + "inputRandomTextCommand": null, + "launchAppCommand": null, + "setPermissionsCommand": null, + "applyConfigurationCommand": null, + "openLinkCommand": null, + "pressKeyCommand": null, + "eraseTextCommand": null, + "hideKeyboardCommand": null, + "takeScreenshotCommand": null, + "stopAppCommand": null, + "killAppCommand": null, + "clearStateCommand": null, + "clearKeychainCommand": null, + "runFlowCommand": null, + "setLocationCommand": null, + "setOrientationCommand": null, + "repeatCommand": null, + "copyTextCommand": null, + "setClipboardCommand": null, + "pasteTextCommand": null, + "defineVariablesCommand": null, + "runScriptCommand": null, + "waitForAnimationToEndCommand": null, + "evalScriptCommand": null, + "scrollUntilVisible": null, + "travelCommand": null, + "startRecordingCommand": null, + "stopRecordingCommand": null, + "addMediaCommand": null, + "setAirplaneModeCommand": null, + "toggleAirplaneModeCommand": null, + "retryCommand": null + } + ], + "condition": null, + "sourceDescription": null, + "config": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/100_tapOn_multiple_times", + "file": "upstream/100_tapOn_multiple_times.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.other.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": "Button", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": { + "repeat": 3, + "delay": 1 + }, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/101_doubleTapOn", + "file": "upstream/101_doubleTapOn.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.other.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": "Button", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": { + "repeat": 2, + "delay": 100 + }, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/114_child_of_selector", + "file": "upstream/114_child_of_selector.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": { + "textRegex": "child_id", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": { + "textRegex": "parent_id_1", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "css": null + }, + "notVisible": null, + "scriptCondition": null, + "label": null + }, + "timeout": null, + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": null, + "notVisible": { + "textRegex": "child_id", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": { + "textRegex": "parent_id_3", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "css": null + }, + "scriptCondition": null, + "label": null + }, + "timeout": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/116_kill_app", + "file": "upstream/116_kill_app.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "KillAppCommand", + "fields": { + "appId": "com.example.app", + "label": null, + "optional": false + } + }, + { + "type": "KillAppCommand", + "fields": { + "appId": "another.app", + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/119_retry_commands", + "file": "upstream/119_retry_commands.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.other.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "RetryCommand", + "fields": { + "maxRetries": "3", + "commands": [ + { + "tapOnElement": null, + "tapOnPoint": null, + "tapOnPointV2Command": null, + "scrollCommand": { + "label": null, + "optional": false + }, + "swipeCommand": null, + "backPressCommand": null, + "assertCommand": null, + "assertConditionCommand": null, + "assertScreenshotCommand": null, + "assertNoDefectsWithAICommand": null, + "assertWithAICommand": null, + "extractTextWithAICommand": null, + "inputTextCommand": null, + "inputRandomTextCommand": null, + "launchAppCommand": null, + "setPermissionsCommand": null, + "applyConfigurationCommand": null, + "openLinkCommand": null, + "pressKeyCommand": null, + "eraseTextCommand": null, + "hideKeyboardCommand": null, + "takeScreenshotCommand": null, + "stopAppCommand": null, + "killAppCommand": null, + "clearStateCommand": null, + "clearKeychainCommand": null, + "runFlowCommand": null, + "setLocationCommand": null, + "setOrientationCommand": null, + "repeatCommand": null, + "copyTextCommand": null, + "setClipboardCommand": null, + "pasteTextCommand": null, + "defineVariablesCommand": null, + "runScriptCommand": null, + "waitForAnimationToEndCommand": null, + "evalScriptCommand": null, + "scrollUntilVisible": null, + "travelCommand": null, + "startRecordingCommand": null, + "stopRecordingCommand": null, + "addMediaCommand": null, + "setAirplaneModeCommand": null, + "toggleAirplaneModeCommand": null, + "retryCommand": null + }, + { + "tapOnElement": { + "selector": { + "textRegex": "Button", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": 40, + "relativePoint": null, + "label": null, + "optional": false + }, + "tapOnPoint": null, + "tapOnPointV2Command": null, + "scrollCommand": null, + "swipeCommand": null, + "backPressCommand": null, + "assertCommand": null, + "assertConditionCommand": null, + "assertScreenshotCommand": null, + "assertNoDefectsWithAICommand": null, + "assertWithAICommand": null, + "extractTextWithAICommand": null, + "inputTextCommand": null, + "inputRandomTextCommand": null, + "launchAppCommand": null, + "setPermissionsCommand": null, + "applyConfigurationCommand": null, + "openLinkCommand": null, + "pressKeyCommand": null, + "eraseTextCommand": null, + "hideKeyboardCommand": null, + "takeScreenshotCommand": null, + "stopAppCommand": null, + "killAppCommand": null, + "clearStateCommand": null, + "clearKeychainCommand": null, + "runFlowCommand": null, + "setLocationCommand": null, + "setOrientationCommand": null, + "repeatCommand": null, + "copyTextCommand": null, + "setClipboardCommand": null, + "pasteTextCommand": null, + "defineVariablesCommand": null, + "runScriptCommand": null, + "waitForAnimationToEndCommand": null, + "evalScriptCommand": null, + "scrollUntilVisible": null, + "travelCommand": null, + "startRecordingCommand": null, + "stopRecordingCommand": null, + "addMediaCommand": null, + "setAirplaneModeCommand": null, + "toggleAirplaneModeCommand": null, + "retryCommand": null + }, + { + "tapOnElement": null, + "tapOnPoint": null, + "tapOnPointV2Command": null, + "scrollCommand": { + "label": null, + "optional": false + }, + "swipeCommand": null, + "backPressCommand": null, + "assertCommand": null, + "assertConditionCommand": null, + "assertScreenshotCommand": null, + "assertNoDefectsWithAICommand": null, + "assertWithAICommand": null, + "extractTextWithAICommand": null, + "inputTextCommand": null, + "inputRandomTextCommand": null, + "launchAppCommand": null, + "setPermissionsCommand": null, + "applyConfigurationCommand": null, + "openLinkCommand": null, + "pressKeyCommand": null, + "eraseTextCommand": null, + "hideKeyboardCommand": null, + "takeScreenshotCommand": null, + "stopAppCommand": null, + "killAppCommand": null, + "clearStateCommand": null, + "clearKeychainCommand": null, + "runFlowCommand": null, + "setLocationCommand": null, + "setOrientationCommand": null, + "repeatCommand": null, + "copyTextCommand": null, + "setClipboardCommand": null, + "pasteTextCommand": null, + "defineVariablesCommand": null, + "runScriptCommand": null, + "waitForAnimationToEndCommand": null, + "evalScriptCommand": null, + "scrollUntilVisible": null, + "travelCommand": null, + "startRecordingCommand": null, + "stopRecordingCommand": null, + "addMediaCommand": null, + "setAirplaneModeCommand": null, + "toggleAirplaneModeCommand": null, + "retryCommand": null + } + ], + "config": null, + "sourceDescription": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/120_tap_on_element_retryTapIfNoChange", + "file": "upstream/120_tap_on_element_retryTapIfNoChange.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": ".*button.*", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": true, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "upstream/131_setPermissions", + "file": "upstream/131_setPermissions.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "SetPermissionsCommand", + "fields": { + "appId": "com.example.app", + "permissions": { + "all": "deny", + "notifications": "unset" + }, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "bug-classes/percent-decimal-swipe", + "file": "bug-classes/percent-decimal-swipe.yaml", + "status": "rejected", + "error": { + "class": "FlowParseException", + "message": "Parsing Failed" + } + }, + { + "id": "bug-classes/retry-over-cap", + "file": "bug-classes/retry-over-cap.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "RetryCommand", + "fields": { + "maxRetries": "99", + "commands": [ + { + "tapOnElement": { + "selector": { + "textRegex": "Retry", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + }, + "tapOnPoint": null, + "tapOnPointV2Command": null, + "scrollCommand": null, + "swipeCommand": null, + "backPressCommand": null, + "assertCommand": null, + "assertConditionCommand": null, + "assertScreenshotCommand": null, + "assertNoDefectsWithAICommand": null, + "assertWithAICommand": null, + "extractTextWithAICommand": null, + "inputTextCommand": null, + "inputRandomTextCommand": null, + "launchAppCommand": null, + "setPermissionsCommand": null, + "applyConfigurationCommand": null, + "openLinkCommand": null, + "pressKeyCommand": null, + "eraseTextCommand": null, + "hideKeyboardCommand": null, + "takeScreenshotCommand": null, + "stopAppCommand": null, + "killAppCommand": null, + "clearStateCommand": null, + "clearKeychainCommand": null, + "runFlowCommand": null, + "setLocationCommand": null, + "setOrientationCommand": null, + "repeatCommand": null, + "copyTextCommand": null, + "setClipboardCommand": null, + "pasteTextCommand": null, + "defineVariablesCommand": null, + "runScriptCommand": null, + "waitForAnimationToEndCommand": null, + "evalScriptCommand": null, + "scrollUntilVisible": null, + "travelCommand": null, + "startRecordingCommand": null, + "stopRecordingCommand": null, + "addMediaCommand": null, + "setAirplaneModeCommand": null, + "toggleAirplaneModeCommand": null, + "retryCommand": null + } + ], + "config": null, + "sourceDescription": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "bug-classes/settle-after-tap", + "file": "bug-classes/settle-after-tap.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": "Submit", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "bug-classes/target-swipe-missing-direction", + "file": "bug-classes/target-swipe-missing-direction.yaml", + "status": "rejected", + "error": { + "class": "FlowParseException", + "message": "Parsing Failed" + } + }, + { + "id": "invalid/bad-swipe-direction", + "file": "invalid/bad-swipe-direction.yaml", + "status": "rejected", + "error": { + "class": "FlowParseException", + "message": "Parsing Failed" + } + }, + { + "id": "invalid/commands-not-a-list", + "file": "invalid/commands-not-a-list.yaml", + "status": "rejected", + "error": { + "class": "FlowParseException", + "message": "Commands Section Required" + } + }, + { + "id": "invalid/duplicate-keys", + "file": "invalid/duplicate-keys.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": "B", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "invalid/malformed-selector", + "file": "invalid/malformed-selector.yaml", + "status": "rejected", + "error": { + "class": "SyntaxError", + "message": "Failed to parse file: /invalid/malformed-selector.yaml" + } + }, + { + "id": "invalid/unknown-command", + "file": "invalid/unknown-command.yaml", + "status": "rejected", + "error": { + "class": "FlowParseException", + "message": "Invalid Command: tapOnn" + } + }, + { + "id": "invalid/unknown-selector-field", + "file": "invalid/unknown-selector-field.yaml", + "status": "rejected", + "error": { + "class": "FlowParseException", + "message": "Unknown Property: bogusField" + } + }, + { + "id": "authored/doubletap", + "file": "authored/doubletap.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": "Button", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": { + "repeat": 2, + "delay": 100 + }, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "authored/extended-wait", + "file": "authored/extended-wait.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": { + "textRegex": null, + "idRegex": "Item", + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "notVisible": null, + "scriptCondition": null, + "label": null + }, + "timeout": "1000", + "label": null, + "optional": false + } + }, + { + "type": "AssertConditionCommand", + "fields": { + "condition": { + "platform": null, + "visible": null, + "notVisible": { + "textRegex": null, + "idRegex": "Another", + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "scriptCondition": null, + "label": null + }, + "timeout": "1000", + "label": null, + "optional": false + } + } + ] + }, + { + "id": "authored/presskey", + "file": "authored/presskey.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "ENTER", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "HOME", + "label": null, + "optional": false + } + }, + { + "type": "PressKeyCommand", + "fields": { + "code": "BACK", + "label": null, + "optional": false + } + } + ] + }, + { + "id": "authored/repeat", + "file": "authored/repeat.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "RepeatCommand", + "fields": { + "times": "3", + "condition": null, + "commands": [ + { + "tapOnElement": { + "selector": { + "textRegex": "Button", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + }, + "tapOnPoint": null, + "tapOnPointV2Command": null, + "scrollCommand": null, + "swipeCommand": null, + "backPressCommand": null, + "assertCommand": null, + "assertConditionCommand": null, + "assertScreenshotCommand": null, + "assertNoDefectsWithAICommand": null, + "assertWithAICommand": null, + "extractTextWithAICommand": null, + "inputTextCommand": null, + "inputRandomTextCommand": null, + "launchAppCommand": null, + "setPermissionsCommand": null, + "applyConfigurationCommand": null, + "openLinkCommand": null, + "pressKeyCommand": null, + "eraseTextCommand": null, + "hideKeyboardCommand": null, + "takeScreenshotCommand": null, + "stopAppCommand": null, + "killAppCommand": null, + "clearStateCommand": null, + "clearKeychainCommand": null, + "runFlowCommand": null, + "setLocationCommand": null, + "setOrientationCommand": null, + "repeatCommand": null, + "copyTextCommand": null, + "setClipboardCommand": null, + "pasteTextCommand": null, + "defineVariablesCommand": null, + "runScriptCommand": null, + "waitForAnimationToEndCommand": null, + "evalScriptCommand": null, + "scrollUntilVisible": null, + "travelCommand": null, + "startRecordingCommand": null, + "stopRecordingCommand": null, + "addMediaCommand": null, + "setAirplaneModeCommand": null, + "toggleAirplaneModeCommand": null, + "retryCommand": null + } + ], + "label": null, + "optional": false + } + } + ] + }, + { + "id": "authored/runflow-main", + "file": "authored/runflow-main.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": "Before", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + } + }, + { + "type": "RunFlowCommand", + "fields": { + "commands": [ + { + "tapOnElement": null, + "tapOnPoint": null, + "tapOnPointV2Command": null, + "scrollCommand": null, + "swipeCommand": null, + "backPressCommand": null, + "assertCommand": null, + "assertConditionCommand": null, + "assertScreenshotCommand": null, + "assertNoDefectsWithAICommand": null, + "assertWithAICommand": null, + "extractTextWithAICommand": null, + "inputTextCommand": null, + "inputRandomTextCommand": null, + "launchAppCommand": null, + "setPermissionsCommand": null, + "applyConfigurationCommand": { + "config": { + "appId": "com.example.include", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + }, + "openLinkCommand": null, + "pressKeyCommand": null, + "eraseTextCommand": null, + "hideKeyboardCommand": null, + "takeScreenshotCommand": null, + "stopAppCommand": null, + "killAppCommand": null, + "clearStateCommand": null, + "clearKeychainCommand": null, + "runFlowCommand": null, + "setLocationCommand": null, + "setOrientationCommand": null, + "repeatCommand": null, + "copyTextCommand": null, + "setClipboardCommand": null, + "pasteTextCommand": null, + "defineVariablesCommand": null, + "runScriptCommand": null, + "waitForAnimationToEndCommand": null, + "evalScriptCommand": null, + "scrollUntilVisible": null, + "travelCommand": null, + "startRecordingCommand": null, + "stopRecordingCommand": null, + "addMediaCommand": null, + "setAirplaneModeCommand": null, + "toggleAirplaneModeCommand": null, + "retryCommand": null + }, + { + "tapOnElement": null, + "tapOnPoint": null, + "tapOnPointV2Command": null, + "scrollCommand": null, + "swipeCommand": null, + "backPressCommand": null, + "assertCommand": null, + "assertConditionCommand": null, + "assertScreenshotCommand": null, + "assertNoDefectsWithAICommand": null, + "assertWithAICommand": null, + "extractTextWithAICommand": null, + "inputTextCommand": null, + "inputRandomTextCommand": null, + "launchAppCommand": { + "appId": "com.example.include", + "clearState": null, + "clearKeychain": null, + "stopApp": null, + "permissions": null, + "launchArguments": null, + "label": null, + "optional": false + }, + "setPermissionsCommand": null, + "applyConfigurationCommand": null, + "openLinkCommand": null, + "pressKeyCommand": null, + "eraseTextCommand": null, + "hideKeyboardCommand": null, + "takeScreenshotCommand": null, + "stopAppCommand": null, + "killAppCommand": null, + "clearStateCommand": null, + "clearKeychainCommand": null, + "runFlowCommand": null, + "setLocationCommand": null, + "setOrientationCommand": null, + "repeatCommand": null, + "copyTextCommand": null, + "setClipboardCommand": null, + "pasteTextCommand": null, + "defineVariablesCommand": null, + "runScriptCommand": null, + "waitForAnimationToEndCommand": null, + "evalScriptCommand": null, + "scrollUntilVisible": null, + "travelCommand": null, + "startRecordingCommand": null, + "stopRecordingCommand": null, + "addMediaCommand": null, + "setAirplaneModeCommand": null, + "toggleAirplaneModeCommand": null, + "retryCommand": null + }, + { + "tapOnElement": { + "selector": { + "textRegex": null, + "idRegex": "included-button", + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + }, + "tapOnPoint": null, + "tapOnPointV2Command": null, + "scrollCommand": null, + "swipeCommand": null, + "backPressCommand": null, + "assertCommand": null, + "assertConditionCommand": null, + "assertScreenshotCommand": null, + "assertNoDefectsWithAICommand": null, + "assertWithAICommand": null, + "extractTextWithAICommand": null, + "inputTextCommand": null, + "inputRandomTextCommand": null, + "launchAppCommand": null, + "setPermissionsCommand": null, + "applyConfigurationCommand": null, + "openLinkCommand": null, + "pressKeyCommand": null, + "eraseTextCommand": null, + "hideKeyboardCommand": null, + "takeScreenshotCommand": null, + "stopAppCommand": null, + "killAppCommand": null, + "clearStateCommand": null, + "clearKeychainCommand": null, + "runFlowCommand": null, + "setLocationCommand": null, + "setOrientationCommand": null, + "repeatCommand": null, + "copyTextCommand": null, + "setClipboardCommand": null, + "pasteTextCommand": null, + "defineVariablesCommand": null, + "runScriptCommand": null, + "waitForAnimationToEndCommand": null, + "evalScriptCommand": null, + "scrollUntilVisible": null, + "travelCommand": null, + "startRecordingCommand": null, + "stopRecordingCommand": null, + "addMediaCommand": null, + "setAirplaneModeCommand": null, + "toggleAirplaneModeCommand": null, + "retryCommand": null + } + ], + "condition": null, + "sourceDescription": "runflow-child.yaml", + "config": { + "appId": "com.example.include", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "TapOnElementCommand", + "fields": { + "selector": { + "textRegex": "After", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "retryIfNoChange": false, + "waitUntilVisible": false, + "longPress": false, + "repeat": null, + "waitToSettleTimeoutMs": null, + "relativePoint": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "authored/runscript", + "file": "authored/runscript.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "RunScriptCommand", + "fields": { + "script": "output.doubled = 2 + 2;\n", + "env": {}, + "sourceDescription": "/authored/runscript.js", + "condition": null, + "label": null, + "optional": false + } + } + ] + }, + { + "id": "authored/scroll-until-visible", + "file": "authored/scroll-until-visible.yaml", + "status": "parsed", + "commands": [ + { + "type": "ApplyConfigurationCommand", + "fields": { + "config": { + "appId": "com.example.app", + "name": null, + "tags": [], + "ext": {}, + "onFlowStart": null, + "onFlowComplete": null, + "properties": {} + }, + "label": null, + "optional": false + } + }, + { + "type": "ScrollUntilVisibleCommand", + "fields": { + "selector": { + "textRegex": "Test", + "idRegex": null, + "size": null, + "below": null, + "above": null, + "leftOf": null, + "rightOf": null, + "containsChild": null, + "containsDescendants": null, + "traits": null, + "index": null, + "enabled": null, + "optional": false, + "selected": null, + "checked": null, + "focused": null, + "childOf": null, + "css": null + }, + "direction": "DOWN", + "scrollDuration": "40", + "visibilityPercentage": 100, + "timeout": "10000", + "waitToSettleTimeoutMs": null, + "centerElement": false, + "originalSpeedValue": "40", + "label": null, + "optional": false, + "visibilityPercentageNormalized": 1 + } + } + ] + } + ], + "contentHash": "7e657d22c33fd784980ca793962fce001d824f2fa9efdc65cf3c95174138b19a" +} diff --git a/scripts/maestro-conformance/fixtures/layer2-semantics.json b/scripts/maestro-conformance/fixtures/layer2-semantics.json new file mode 100644 index 000000000..3e124261c --- /dev/null +++ b/scripts/maestro-conformance/fixtures/layer2-semantics.json @@ -0,0 +1,70 @@ +{ + "schemaVersion": 2, + "generatedBy": "scripts/maestro-conformance/regenerate.mjs", + "upstream": { + "project": "mobile-dev-inc/Maestro", + "version": "2.5.1", + "tag": "v2.5.1", + "commit": "a4c7c95f5ba1884858f7e35efa6b8e0165db9448", + "artifacts": [ + { + "coordinate": "dev.mobile:maestro-orchestra:2.5.1", + "role": "Layer 1 YAML parser (YamlCommandReader) and Layer 2 retry/erase constants (Orchestra).", + "sha256": "bbd8e5e35e3c706c6d4760bb1026162474b171155a5fa76f6de1360ab3705468" + }, + { + "coordinate": "dev.mobile:maestro-orchestra-models:2.5.1", + "role": "Command model classes and parser-observed defaults (Layer 2 model defaults).", + "sha256": "0c147f7dff3704e1e04cb728339f2637a6086c234d7984c1434536680f6749ee" + }, + { + "coordinate": "dev.mobile:maestro-client:2.5.1", + "role": "Layer 2 driver constants (Maestro.SCREENSHOT_DIFF_THRESHOLD/ANIMATION_TIMEOUT_MS, IOSDriver.SCREEN_SETTLE_TIMEOUT_MS).", + "sha256": "a6b8c63c0858e8e256752f617fd27b35b0be48e78d367f9cbdffa735a6fcc991" + } + ] + }, + "layer": 2, + "description": "Upstream Maestro semantic vectors. Generated by the JVM harness; do not hand-edit.", + "constants": [ + { + "id": "retryMaxRetries", + "symbol": "maestro.orchestra.Orchestra#MAX_RETRIES_ALLOWED", + "note": "Upper bound Orchestra clamps a retry block's maxRetries to (coerceAtMost).", + "value": 3 + }, + { + "id": "maxEraseCharacters", + "symbol": "maestro.orchestra.Orchestra#MAX_ERASE_CHARACTERS", + "note": "Upper bound on eraseText character count.", + "value": 50 + }, + { + "id": "animationWaitThreshold", + "symbol": "maestro.Maestro#SCREENSHOT_DIFF_THRESHOLD", + "note": "Screenshot-diff threshold (romankh3 0-100 scale) for waitForAnimationToEnd stability.", + "value": 0.005 + }, + { + "id": "animationWaitTimeoutMs", + "symbol": "maestro.Maestro#ANIMATION_TIMEOUT_MS", + "note": "Default waitForAnimationToEnd timeout.", + "value": 15000 + }, + { + "id": "iosScreenSettleTimeoutMs", + "symbol": "maestro.drivers.IOSDriver#SCREEN_SETTLE_TIMEOUT_MS", + "note": "Upstream iOS pre-tap static-screen gate; agent-device intentionally omits it (expected divergence).", + "value": 3000 + } + ], + "modelDefaults": [ + { + "id": "swipeDurationMs", + "flow": "- swipe:\n direction: LEFT", + "accessor": "SwipeCommand.duration", + "value": 400 + } + ], + "contentHash": "d53c6914979adc45f4ae8146411f078ab2db859a33261fec2546449e3a78b702" +} diff --git a/scripts/maestro-conformance/jvm-harness/.gitignore b/scripts/maestro-conformance/jvm-harness/.gitignore new file mode 100644 index 000000000..00f6d67f2 --- /dev/null +++ b/scripts/maestro-conformance/jvm-harness/.gitignore @@ -0,0 +1,5 @@ +# Gradle/Kotlin build output for the conformance harness. The harness only runs +# during regeneration (regenerate.mjs); nothing here is checked in except sources +# and the committed wrapper. +.gradle/ +build/ diff --git a/scripts/maestro-conformance/jvm-harness/build.gradle.kts b/scripts/maestro-conformance/jvm-harness/build.gradle.kts new file mode 100644 index 000000000..a7fc7703a --- /dev/null +++ b/scripts/maestro-conformance/jvm-harness/build.gradle.kts @@ -0,0 +1,48 @@ +plugins { + kotlin("jvm") version "2.2.20" + application +} + +// Pins for the upstream Maestro artifacts the oracle is generated against. These +// MUST match `pinned-upstream.json` in the parent directory; `regenerate.mjs` +// verifies the resolved jar SHA-256s against that file before trusting output. +val maestroVersion = "2.5.1" + +repositories { + mavenCentral() +} + +dependencies { + // Layer 1 parser + Layer 2 model defaults (YamlCommandReader, command models). + implementation("dev.mobile:maestro-orchestra:$maestroVersion") + implementation("dev.mobile:maestro-orchestra-models:$maestroVersion") + // Layer 2 constants (Maestro.SCREENSHOT_DIFF_THRESHOLD / ANIMATION_TIMEOUT_MS, + // Orchestra.MAX_RETRIES_ALLOWED). Read as bytecode ConstantValue attributes via + // ASM — the harness never initializes these driver-bound classes. + implementation("dev.mobile:maestro-client:$maestroVersion") + implementation("org.ow2.asm:asm:9.7") + // Jackson (already transitive via maestro-orchestra) for JSON emission. + implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.17.2") +} + +application { + mainClass.set("dev.agentdevice.conformance.MainKt") +} + +kotlin { + jvmToolchain(17) +} + +// Prints the resolved upstream (dev.mobile) jars so `regenerate.mjs` can verify +// the SHA-256 of the exact bytes the harness compiled and ran against, against +// `pinned-upstream.json`. This is the regeneration-time artifact-integrity gate. +tasks.register("printUpstreamJars") { + val artifacts = configurations.named("runtimeClasspath") + doLast { + artifacts.get().resolvedConfiguration.resolvedArtifacts + .filter { it.moduleVersion.id.group == "dev.mobile" } + .forEach { a -> + println("UPSTREAM_JAR ${a.moduleVersion.id.name}:${a.moduleVersion.id.version} ${a.file}") + } + } +} diff --git a/scripts/maestro-conformance/jvm-harness/gradle/wrapper/gradle-wrapper.jar b/scripts/maestro-conformance/jvm-harness/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..a4b76b953 Binary files /dev/null and b/scripts/maestro-conformance/jvm-harness/gradle/wrapper/gradle-wrapper.jar differ diff --git a/scripts/maestro-conformance/jvm-harness/gradle/wrapper/gradle-wrapper.properties b/scripts/maestro-conformance/jvm-harness/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..b2870d611 --- /dev/null +++ b/scripts/maestro-conformance/jvm-harness/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,11 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip +# Published checksum for gradle-8.10.2-bin.zip. This tool refuses to trust an +# unverified upstream jar; the toolchain that generates the fixtures gets the +# same treatment. Update alongside distributionUrl on a Gradle bump. +distributionSha256Sum=31c55713e40233a8303827ceb42ca48a47267a0ad4bab9177123121e71524c26 +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/scripts/maestro-conformance/jvm-harness/gradlew b/scripts/maestro-conformance/jvm-harness/gradlew new file mode 100755 index 000000000..f5feea6d6 --- /dev/null +++ b/scripts/maestro-conformance/jvm-harness/gradlew @@ -0,0 +1,252 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/scripts/maestro-conformance/jvm-harness/gradlew.bat b/scripts/maestro-conformance/jvm-harness/gradlew.bat new file mode 100644 index 000000000..9b42019c7 --- /dev/null +++ b/scripts/maestro-conformance/jvm-harness/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/scripts/maestro-conformance/jvm-harness/settings.gradle.kts b/scripts/maestro-conformance/jvm-harness/settings.gradle.kts new file mode 100644 index 000000000..3ab36d7a0 --- /dev/null +++ b/scripts/maestro-conformance/jvm-harness/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "maestro-conformance-harness" diff --git a/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/ClassConstants.kt b/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/ClassConstants.kt new file mode 100644 index 000000000..d328b2d69 --- /dev/null +++ b/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/ClassConstants.kt @@ -0,0 +1,41 @@ +package dev.agentdevice.conformance + +import org.objectweb.asm.ClassReader +import org.objectweb.asm.ClassVisitor +import org.objectweb.asm.FieldVisitor +import org.objectweb.asm.Opcodes + +/** + * Reads a `static final` field's compile-time constant straight out of the + * pinned bytecode via its `ConstantValue` attribute. This never loads or + * initializes the declaring class, so it is safe against `maestro.Maestro` / + * `maestro.drivers.*Driver`, whose `` pulls in driver/native state. + * + * The value is genuinely extracted from the resolved jar — not transcribed — + * which is the whole point of the semantic-vector layer. + */ +fun readConstant(internalClassName: String, fieldName: String): Any { + val resource = "$internalClassName.class" + val bytes = Thread.currentThread().contextClassLoader.getResourceAsStream(resource) + ?.use { it.readBytes() } + ?: error("Class resource not found on the pinned classpath: $resource") + + var found: Any? = null + ClassReader(bytes).accept(object : ClassVisitor(Opcodes.ASM9) { + override fun visitField( + access: Int, + name: String, + descriptor: String, + signature: String?, + value: Any?, + ): FieldVisitor? { + if (name == fieldName) { + found = value + ?: error("Field $internalClassName#$fieldName has no ConstantValue attribute.") + } + return null + } + }, ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG or ClassReader.SKIP_FRAMES) + + return found ?: error("Field $internalClassName#$fieldName not found in the pinned bytecode.") +} diff --git a/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/Layer1.kt b/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/Layer1.kt new file mode 100644 index 000000000..9ec1fd468 --- /dev/null +++ b/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/Layer1.kt @@ -0,0 +1,53 @@ +package dev.agentdevice.conformance + +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.ObjectNode +import maestro.orchestra.yaml.YamlCommandReader +import java.nio.file.Path + +/** + * Layer 1 — parser normalization generated from upstream. + * + * Drives the pinned `YamlCommandReader` over every corpus flow and captures, + * per flow, either the faithful parsed command list or the upstream rejection + * (exception class + message). The Node verifier compares our engine's parse of + * the same flow against this generated capture and classifies each flow as + * parses-identically / explicitly-rejected / MISMATCH. + */ +fun emitLayer1(corpusDir: Path, flows: List): ObjectNode { + val root = fixtureMapper.createObjectNode() + root.put("layer", 1) + root.put("description", "Upstream Maestro parser capture. Generated by the JVM harness; do not hand-edit.") + val flowsNode: ArrayNode = root.putArray("flows") + + for (flow in flows) { + val node = flowsNode.addObject() + node.put("id", flow.id) + node.put("file", flow.file) + val path = corpusDir.resolve(flow.file) + try { + val commands = YamlCommandReader.readCommands(path) + node.put("status", "parsed") + val commandsNode = node.putArray("commands") + for (command in commands) { + val active = activeCommand(command) + ?: error("MaestroCommand with no active sub-command in ${flow.file}") + val entry = commandsNode.addObject() + entry.put("type", active.javaClass.simpleName) + entry.set("fields", fixtureMapper.valueToTree(active)) + } + } catch (t: Throwable) { + node.put("status", "rejected") + val error = node.putObject("error") + error.put("class", rootCause(t).javaClass.simpleName) + error.put("message", (rootCause(t).message ?: "").lineSequence().first().trim()) + } + } + return root +} + +private fun rootCause(t: Throwable): Throwable { + var current = t + while (current.cause != null && current.cause !== current) current = current.cause!! + return current +} diff --git a/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/Layer2.kt b/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/Layer2.kt new file mode 100644 index 000000000..a2d55502a --- /dev/null +++ b/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/Layer2.kt @@ -0,0 +1,124 @@ +package dev.agentdevice.conformance + +import com.fasterxml.jackson.databind.node.ObjectNode +import maestro.orchestra.yaml.YamlCommandReader +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.createTempDirectory + +/** + * Layer 2 — semantic vectors generated by the JVM harness. + * + * Two kinds of vector, both read from the pinned artifacts (never transcribed): + * - `constants`: `static final` values read straight out of the bytecode via + * ASM (no class initialization), e.g. the retry cap and animation-wait + * thresholds. + * - `modelDefaults`: fields defaulted by the upstream parser, observed by + * parsing a minimal inline flow and reading the value back off the command + * model (e.g. the 400ms default swipe duration). + * + * The Node verifier cross-checks each vector against the corresponding + * agent-device engine constant, or records it as a documented reference value + * (for intentionally divergent upstream constants such as the iOS pre-tap gate). + */ +fun emitLayer2(): ObjectNode { + val root = fixtureMapper.createObjectNode() + root.put("layer", 2) + root.put("description", "Upstream Maestro semantic vectors. Generated by the JVM harness; do not hand-edit.") + + val constants = root.putArray("constants") + for (c in CONSTANT_VECTORS) { + val (owner, field) = c.symbol.split("#", limit = 2) + val value = readConstant(owner.replace('.', '/'), field) + val node = constants.addObject() + node.put("id", c.id) + node.put("symbol", c.symbol) + node.put("note", c.note) + putNumber(node, "value", value) + } + + val modelDefaults = root.putArray("modelDefaults") + for (d in MODEL_DEFAULT_VECTORS) { + val value = d.read() + val node = modelDefaults.addObject() + node.put("id", d.id) + node.put("flow", d.flow.trim()) + node.put("accessor", d.accessor) + putNumber(node, "value", value) + } + return root +} + +private data class ConstantVector(val id: String, val symbol: String, val note: String) + +private val CONSTANT_VECTORS = listOf( + ConstantVector( + "retryMaxRetries", + "maestro.orchestra.Orchestra#MAX_RETRIES_ALLOWED", + "Upper bound Orchestra clamps a retry block's maxRetries to (coerceAtMost).", + ), + ConstantVector( + "maxEraseCharacters", + "maestro.orchestra.Orchestra#MAX_ERASE_CHARACTERS", + "Upper bound on eraseText character count.", + ), + ConstantVector( + "animationWaitThreshold", + "maestro.Maestro#SCREENSHOT_DIFF_THRESHOLD", + "Screenshot-diff threshold (romankh3 0-100 scale) for waitForAnimationToEnd stability.", + ), + ConstantVector( + "animationWaitTimeoutMs", + "maestro.Maestro#ANIMATION_TIMEOUT_MS", + "Default waitForAnimationToEnd timeout.", + ), + ConstantVector( + "iosScreenSettleTimeoutMs", + "maestro.drivers.IOSDriver#SCREEN_SETTLE_TIMEOUT_MS", + "Upstream iOS pre-tap static-screen gate; agent-device intentionally omits it (expected divergence).", + ), +) + +private class ModelDefaultVector( + val id: String, + val flow: String, + val accessor: String, + val read: () -> Any, +) + +private val MODEL_DEFAULT_VECTORS = listOf( + ModelDefaultVector( + id = "swipeDurationMs", + flow = "- swipe:\n direction: LEFT\n", + accessor = "SwipeCommand.duration", + read = { parseSingle("- swipe:\n direction: LEFT\n", "getSwipeCommand", "getDuration") }, + ), +) + +/** Parse a one-command flow and read a numeric field off the active command. */ +private fun parseSingle(flowBody: String, wrapperGetter: String, fieldGetter: String): Any { + val dir = createTempDirectory("maestro-conformance-l2") + try { + val file = dir.resolve("flow.yaml") + Files.writeString(file, "appId: com.example\n---\n$flowBody") + val commands = YamlCommandReader.readCommands(file) + for (command in commands) { + val sub = command.javaClass.getMethod(wrapperGetter).invoke(command) ?: continue + return sub.javaClass.getMethod(fieldGetter).invoke(sub) + ?: error("$wrapperGetter.$fieldGetter returned null") + } + error("No $wrapperGetter command produced by flow") + } finally { + Files.walk(dir).sorted(Comparator.reverseOrder()).forEach(Files::delete) + } +} + +private fun putNumber(node: ObjectNode, key: String, value: Any) { + when (value) { + is Int -> node.put(key, value) + is Long -> node.put(key, value) + is Double -> node.put(key, value) + is Float -> node.put(key, value.toDouble()) + else -> node.put(key, value.toString()) + } +} diff --git a/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/Main.kt b/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/Main.kt new file mode 100644 index 000000000..29db26e3e --- /dev/null +++ b/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/Main.kt @@ -0,0 +1,62 @@ +package dev.agentdevice.conformance + +import java.nio.file.Files +import java.nio.file.Path + +data class CorpusFlow(val id: String, val file: String) + +/** + * Conformance harness entry point. + * + * Usage: + * run --args="--corpus --out " + * + * Emits `/layer1-parser.json` and `/layer2-semantics.json` + * containing only the generated content. `regenerate.mjs` verifies the resolved + * jar SHA-256s and wraps each file with the upstream pin before checking it in. + */ +fun main(args: Array) { + val options = parseArgs(args) + val corpusDir = options.getValue("corpus").let(Path::of) + val outDir = options.getValue("out").let(Path::of) + Files.createDirectories(outDir) + + val flows = readManifest(corpusDir.resolve("manifest.json")) + + val layer1 = emitLayer1(corpusDir, flows) + // The parser resolves runScript/runFlow file references to absolute paths; + // rewrite the corpus root to a stable token so fixtures are machine-independent. + writeFixture(outDir.resolve("layer1-parser.json"), layer1, corpusDir.toAbsolutePath().toString()) + + val layer2 = emitLayer2() + writeFixture(outDir.resolve("layer2-semantics.json"), layer2, null) + + System.err.println("Emitted layer1 (${flows.size} flows) and layer2 to $outDir") +} + +private fun parseArgs(args: Array): Map { + val result = mutableMapOf() + var i = 0 + while (i < args.size) { + val arg = args[i] + require(arg.startsWith("--")) { "Unexpected argument: $arg" } + require(i + 1 < args.size) { "Missing value for $arg" } + result[arg.removePrefix("--")] = args[i + 1] + i += 2 + } + return result +} + +private fun readManifest(manifestPath: Path): List { + val root = fixtureMapper.readTree(Files.readString(manifestPath)) + val flows = root.get("flows") ?: error("manifest.json missing 'flows'") + return flows + .filterNot { it.path("includeTargetOnly").asBoolean(false) } + .map { CorpusFlow(it.get("id").asText(), it.get("file").asText()) } +} + +private fun writeFixture(path: Path, node: com.fasterxml.jackson.databind.JsonNode, corpusRoot: String?) { + var json = fixtureMapper.writeValueAsString(node) + if (corpusRoot != null) json = json.replace(corpusRoot, "") + Files.writeString(path, json + "\n") +} diff --git a/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/Serialization.kt b/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/Serialization.kt new file mode 100644 index 000000000..a0fb5c902 --- /dev/null +++ b/scripts/maestro-conformance/jvm-harness/src/main/kotlin/dev/agentdevice/conformance/Serialization.kt @@ -0,0 +1,35 @@ +package dev.agentdevice.conformance + +import com.fasterxml.jackson.databind.SerializationFeature +import com.fasterxml.jackson.databind.json.JsonMapper +import com.fasterxml.jackson.module.kotlin.kotlinModule + +/** + * A single JSON mapper used for every fixture we emit. Keys are sorted so the + * generated fixtures are byte-stable across regenerations (the deterministic + * verifier and `regenerate.mjs` both rely on stable ordering). + */ +val fixtureMapper: JsonMapper = JsonMapper.builder() + .addModule(kotlinModule()) + .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) + .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) + .enable(SerializationFeature.INDENT_OUTPUT) + .build() + +/** + * The single sub-command carried by a Maestro `MaestroCommand` wrapper. The + * wrapper exposes one non-null `getXxxCommand()` / `getTapOnElement()` getter; + * we return the first payload whose class is an `maestro.orchestra.*Command`. + * This is a faithful projection: we do not rename or reshape any field, we only + * locate the active variant. The field-level mapping to our IR lives in the + * checked-in Node normalizer, applied to this generated capture. + */ +fun activeCommand(command: Any): Any? = + command.javaClass.methods + .asSequence() + .filter { it.name.startsWith("get") && it.parameterCount == 0 && it.name != "getClass" } + .mapNotNull { m -> runCatching { m.invoke(command) }.getOrNull() } + .firstOrNull { + it.javaClass.name.startsWith("maestro.orchestra.") && + it.javaClass.simpleName.endsWith("Command") + } diff --git a/scripts/maestro-conformance/normalize.ts b/scripts/maestro-conformance/normalize.ts new file mode 100644 index 000000000..463327b5d --- /dev/null +++ b/scripts/maestro-conformance/normalize.ts @@ -0,0 +1,458 @@ +// Canonical projection shared by both engines. Layer-1 conformance compares the +// upstream parser capture (generated) and our engine's live parse after both are +// projected into this representation. It captures the conformance-critical +// essence of each command — identity, selectors, geometry, gesture direction, +// retry/repeat counts, launch flags — and deliberately drops representation-only +// differences (regex-vs-literal selector storage, runScript path-vs-content, +// nested runFlow expansion) that the two IR designs express differently. + +import type { + MaestroCommand, + MaestroGestureTarget, + MaestroProgram, + MaestroSelector, + MaestroSwipeGesture, +} from '../../src/compat/maestro/program-ir.ts'; +import { MAESTRO_COMPATIBILITY_PRESETS } from '../../src/compat/maestro/compatibility-policy.ts'; + +export type CanonicalSelector = { + text?: string; + id?: string; + index?: number; + enabled?: boolean; + selected?: boolean; + childOf?: CanonicalSelector; +}; + +export type CanonicalPoint = { x: number; y: number; unit: 'px' | 'percent'; expr?: string }; + +export type CanonicalTarget = { + selector?: CanonicalSelector; + point?: CanonicalPoint; +}; + +export type CanonicalGesture = + | { mode: 'direction'; direction: string; duration?: number } + | { mode: 'coordinates'; start?: CanonicalPoint; end?: CanonicalPoint; duration?: number } + | { mode: 'element'; from: CanonicalSelector; direction?: string; duration?: number }; + +export type CanonicalCommand = + | { kind: 'launchApp'; appId?: string; clearState?: boolean; stopApp?: boolean } + // Upstream models `doubleTapOn` as a tap with repeat.repeat == 2, so the repeat + // COUNT is the canonical field on both sides rather than a `double` variant on + // one — that keeps our distinct tapOn/doubleTapOn kinds comparable to upstream + // and preserves conformance signal for tapOn.repeat/delay. + | { kind: 'tap'; longPress: boolean; repeat: number; delay?: number; target: CanonicalTarget } + | { kind: 'assert'; mode: 'visible' | 'notVisible'; timed: boolean; selector?: CanonicalSelector } + | { kind: 'swipe'; gesture: CanonicalGesture } + | { kind: 'inputText'; text?: string } + | { kind: 'eraseText'; count?: number } + | { kind: 'openLink'; link?: string } + | { kind: 'scroll' } + | { kind: 'scrollUntilVisible'; direction?: string; selector?: CanonicalSelector } + | { kind: 'pressKey'; key: string } + | { kind: 'back' } + | { kind: 'hideKeyboard' } + | { kind: 'takeScreenshot' } + | { kind: 'waitForAnimationToEnd'; timeout?: number } + | { kind: 'stopApp' } + | { kind: 'repeat'; times: string | number } + | { kind: 'retry'; maxRetries?: string | number } + | { kind: 'runFlow'; source: 'file' | 'commands' } + | { kind: 'runScript' } + | { kind: 'unsupported'; command: string }; + +// --------------------------------------------------------------------------- +// Upstream (generated capture) → canonical +// --------------------------------------------------------------------------- + +/** Upstream command-model types that carry flow config, not a runnable step. */ +const UPSTREAM_CONFIG_TYPES = new Set(['ApplyConfigurationCommand', 'DefineVariablesCommand']); + +type UpstreamCommand = { type: string; fields: Record }; + +export function canonicalizeUpstreamFlow(commands: UpstreamCommand[]): CanonicalCommand[] { + return commands + .filter((command) => !UPSTREAM_CONFIG_TYPES.has(command.type)) + .map(canonicalizeUpstreamCommand); +} + +function canonicalizeUpstreamCommand(command: UpstreamCommand): CanonicalCommand { + const f = command.fields; + switch (command.type) { + case 'LaunchAppCommand': + return dropUndefined({ + kind: 'launchApp', + appId: str(f.appId), + clearState: bool(f.clearState), + stopApp: bool(f.stopApp), + }); + case 'TapOnElementCommand': { + const repeat = asRecord(f.repeat); + return canonicalTap({ + longPress: bool(f.longPress) ?? false, + repeat: num(repeat?.repeat) ?? 1, + delay: num(repeat?.delay), + target: { selector: upstreamSelector(f.selector) }, + }); + } + case 'TapOnPointV2Command': + case 'TapOnPointCommand': + return canonicalTap({ + longPress: false, + repeat: 1, + target: { point: upstreamPoint(f.point) }, + }); + case 'AssertConditionCommand': { + // Upstream serializes every condition slot; the active one is non-null. + const condition = asRecord(f.condition) ?? {}; + if (condition.visible != null) { + return dropUndefined({ + kind: 'assert', + mode: 'visible', + timed: f.timeout != null, + selector: upstreamSelector(condition.visible), + }); + } + if (condition.notVisible != null) { + return dropUndefined({ + kind: 'assert', + mode: 'notVisible', + timed: f.timeout != null, + selector: upstreamSelector(condition.notVisible), + }); + } + return { kind: 'unsupported', command: 'assertTrue' }; + } + case 'SwipeCommand': + return { kind: 'swipe', gesture: upstreamGesture(f) }; + case 'ScrollCommand': + return { kind: 'scroll' }; + case 'ScrollUntilVisibleCommand': + return dropUndefined({ + kind: 'scrollUntilVisible', + direction: lower(str(f.direction)), + selector: upstreamSelector(f.selector), + }); + case 'InputTextCommand': + return dropUndefined({ kind: 'inputText', text: str(f.text) }); + case 'EraseTextCommand': + return dropUndefined({ kind: 'eraseText', count: num(f.charactersToErase) }); + case 'OpenLinkCommand': + return dropUndefined({ kind: 'openLink', link: str(f.link) }); + case 'PressKeyCommand': + return { kind: 'pressKey', key: lower(str(f.code)) ?? '' }; + case 'BackPressCommand': + return { kind: 'back' }; + case 'HideKeyboardCommand': + return { kind: 'hideKeyboard' }; + case 'TakeScreenshotCommand': + return { kind: 'takeScreenshot' }; + case 'WaitForAnimationToEndCommand': + return dropUndefined({ kind: 'waitForAnimationToEnd', timeout: numLike(f.timeout) }); + case 'StopAppCommand': + return { kind: 'stopApp' }; + case 'RepeatCommand': + return { kind: 'repeat', times: numLike(f.times) ?? str(f.times) ?? '' }; + case 'RetryCommand': + return dropUndefined({ kind: 'retry', maxRetries: numLike(f.maxRetries) ?? str(f.maxRetries) }); + case 'RunFlowCommand': + return { kind: 'runFlow', source: f.sourceDescription != null ? 'file' : 'commands' }; + case 'RunScriptCommand': + return { kind: 'runScript' }; + default: + return { kind: 'unsupported', command: unsupportedName(command.type) }; + } +} + +/** + * Build a canonical tap from the effective repeat semantics. `delay` only means + * anything for a repeated tap, so it is dropped for a single tap to keep the two + * engines' representations comparable. + */ +function canonicalTap(tap: { + longPress: boolean; + repeat: number; + delay?: number; + target: CanonicalTarget; +}): CanonicalCommand { + return dropUndefined({ + kind: 'tap', + longPress: tap.longPress, + repeat: tap.repeat, + delay: tap.repeat > 1 ? tap.delay : undefined, + target: tap.target, + }); +} + +function upstreamSelector(value: unknown): CanonicalSelector | undefined { + const record = asRecord(value); + if (!record) return undefined; + return dropUndefined({ + text: str(record.textRegex), + id: str(record.idRegex), + index: numLike(record.index), + enabled: bool(record.enabled), + selected: bool(record.selected), + childOf: upstreamSelector(record.childOf), + }); +} + +/** + * A point the upstream parser accepted but we cannot canonicalize is a hole in + * the oracle, not a value to drop: silently returning undefined would erase the + * point from BOTH sides of the comparison and make an unequal pair compare equal. + * Fail loudly so the projection gets fixed instead. + */ +function upstreamPoint(value: unknown): CanonicalPoint | undefined { + const text = str(value); + if (!text) return undefined; + const match = /^\s*(-?\d+)(%?)\s*,\s*(-?\d+)(%?)\s*$/.exec(text); + if (!match) { + throw new Error( + `Cannot canonicalize upstream point ${JSON.stringify(text)}; extend upstreamPoint() in normalize.ts.`, + ); + } + const unit = match[2] === '%' || match[4] === '%' ? 'percent' : 'px'; + return { x: Number(match[1]), y: Number(match[3]), unit }; +} + +function upstreamGesture(f: Record): CanonicalGesture { + const duration = num(f.duration); + if (f.elementSelector != null) { + return dropUndefined({ + mode: 'element', + from: upstreamSelector(f.elementSelector) ?? {}, + direction: lower(str(f.direction)), + duration, + }); + } + if (f.direction != null && f.startRelative == null && f.startPoint == null) { + return dropUndefined({ mode: 'direction', direction: lower(str(f.direction)) ?? '', duration }); + } + return dropUndefined({ + mode: 'coordinates', + start: pointFromRelativeOrPoint(f.startRelative, f.startPoint), + end: pointFromRelativeOrPoint(f.endRelative, f.endPoint), + duration, + }); +} + +function pointFromRelativeOrPoint(relative: unknown, point: unknown): CanonicalPoint | undefined { + const rel = str(relative); + if (rel) { + const match = /^\s*(\d+)%\s*,\s*(\d+)%\s*$/.exec(rel); + if (match) return { x: Number(match[1]), y: Number(match[2]), unit: 'percent' }; + } + // Absolute swipe endpoints are captured as Point objects, not "x,y" strings. + const record = asRecord(point); + if (record && typeof record.x === 'number' && typeof record.y === 'number') { + return { x: record.x, y: record.y, unit: 'px' }; + } + return undefined; +} + +function unsupportedName(type: string): string { + // e.g. CopyTextFromCommand -> copyTextFrom + const base = type.replace(/Command$/, ''); + return base.charAt(0).toLowerCase() + base.slice(1); +} + +// --------------------------------------------------------------------------- +// Small coercion helpers (upstream stores several fields as strings) +// --------------------------------------------------------------------------- + +function asRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined; +} +function str(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} +function bool(value: unknown): boolean | undefined { + return typeof value === 'boolean' ? value : undefined; +} +function num(value: unknown): number | undefined { + return typeof value === 'number' ? value : undefined; +} +function lower(value: string | undefined): string | undefined { + return value?.toLowerCase(); +} +/** Coerce a numeric-or-string field to a number when it is a plain integer. */ +function numLike(value: unknown): number | undefined { + if (typeof value === 'number') return value; + if (typeof value === 'string' && /^\d+$/.test(value)) return Number(value); + return undefined; +} + +export function dropUndefined>(value: T): T { + for (const key of Object.keys(value)) { + if (value[key] === undefined) delete value[key]; + } + return value; +} + +// --------------------------------------------------------------------------- +// agent-device engine IR → canonical +// --------------------------------------------------------------------------- + +// Upstream materializes these defaults onto the command at parse time (config +// appId onto a bare launchApp, the 400ms swipe duration); our engine defers them +// to execution. Materialize them here so the comparison is on effective values. +const AGENT_SWIPE_DEFAULT_DURATION = MAESTRO_COMPATIBILITY_PRESETS.command.swipeDurationMs; +const AGENT_REPEAT_DELAY_MS = MAESTRO_COMPATIBILITY_PRESETS.command.repeatDelayMs; + +export function canonicalizeAgentCommands( + program: Pick, +): CanonicalCommand[] { + return program.commands.map((command) => canonicalizeAgentCommand(command, program.config)); +} + +function canonicalizeAgentCommand( + command: MaestroCommand, + config: MaestroProgram['config'], +): CanonicalCommand { + switch (command.kind) { + case 'launchApp': + return dropUndefined({ + kind: 'launchApp', + appId: command.appId ?? config.appId, + clearState: command.clearState, + stopApp: command.stopApp, + }); + case 'tapOn': { + const repeat = command.repeat ?? 1; + return canonicalTap({ + longPress: false, + repeat, + delay: repeat > 1 ? (command.delay ?? AGENT_REPEAT_DELAY_MS) : undefined, + target: agentTarget(command.target, command.index, command.childOf), + }); + } + case 'doubleTapOn': + // Upstream compiles doubleTapOn to a repeat-2 tap with the same default delay. + return canonicalTap({ + longPress: false, + repeat: 2, + delay: command.delay ?? AGENT_REPEAT_DELAY_MS, + target: agentTarget(command.target), + }); + case 'longPressOn': + return canonicalTap({ longPress: true, repeat: 1, target: agentTarget(command.target) }); + case 'assertVisible': + return dropUndefined({ + kind: 'assert', + mode: 'visible', + timed: false, + selector: agentSelector(command.target), + }); + case 'assertNotVisible': + return dropUndefined({ + kind: 'assert', + mode: 'notVisible', + timed: false, + selector: agentSelector(command.target), + }); + case 'extendedWaitUntil': + return dropUndefined({ + kind: 'assert', + mode: command.notVisible ? 'notVisible' : 'visible', + timed: true, + selector: agentSelector(command.notVisible ?? command.visible), + }); + case 'swipe': + return { kind: 'swipe', gesture: agentGesture(command.gesture) }; + case 'inputText': + return dropUndefined({ kind: 'inputText', text: command.text }); + case 'eraseText': + return dropUndefined({ kind: 'eraseText', count: command.charactersToErase }); + case 'openLink': + return dropUndefined({ kind: 'openLink', link: command.link }); + case 'scroll': + return { kind: 'scroll' }; + case 'scrollUntilVisible': + return dropUndefined({ + kind: 'scrollUntilVisible', + direction: command.direction, + selector: agentSelector(command.element), + }); + case 'pressKey': + return { kind: 'pressKey', key: command.key.toLowerCase() }; + case 'back': + return { kind: 'back' }; + case 'hideKeyboard': + return { kind: 'hideKeyboard' }; + case 'takeScreenshot': + return { kind: 'takeScreenshot' }; + case 'waitForAnimationToEnd': + return dropUndefined({ kind: 'waitForAnimationToEnd', timeout: command.timeout }); + case 'stopApp': + return { kind: 'stopApp' }; + case 'repeat': + return { kind: 'repeat', times: command.times }; + case 'retry': + return dropUndefined({ kind: 'retry', maxRetries: command.maxRetries }); + case 'runFlow': + return { kind: 'runFlow', source: command.include.kind === 'file' ? 'file' : 'commands' }; + case 'runScript': + return { kind: 'runScript' }; + default: { + const exhaustive: never = command; + throw new Error(`Unhandled agent command: ${JSON.stringify(exhaustive)}`); + } + } +} + +function agentTarget( + target: MaestroGestureTarget, + index?: number, + childOf?: MaestroSelector, +): CanonicalTarget { + if (target.space === 'target') { + return { + selector: dropUndefined({ + ...agentSelector(target.selector), + index, + childOf: childOf ? agentSelector(childOf) : undefined, + }), + }; + } + return { point: { x: target.x, y: target.y, unit: target.space === 'percent' ? 'percent' : 'px' } }; +} + +function agentSelector(selector: MaestroSelector | undefined): CanonicalSelector | undefined { + if (!selector) return undefined; + return dropUndefined({ + text: selector.text, + id: selector.id, + enabled: selector.enabled, + selected: selector.selected, + }); +} + +function agentGesture(gesture: MaestroSwipeGesture): CanonicalGesture { + const duration = gesture.duration ?? AGENT_SWIPE_DEFAULT_DURATION; + switch (gesture.kind) { + case 'screen': + return { mode: 'direction', direction: gesture.direction, duration }; + case 'coordinates': + return dropUndefined({ + mode: 'coordinates', + start: agentPoint(gesture.start), + end: agentPoint(gesture.end), + duration, + }); + case 'target': + return dropUndefined({ + mode: 'element', + from: agentSelector(gesture.from) ?? {}, + direction: gesture.direction, + duration, + }); + } +} + +function agentPoint(coordinate: { space: 'absolute' | 'percent'; x: number; y: number }): CanonicalPoint { + return { x: coordinate.x, y: coordinate.y, unit: coordinate.space === 'percent' ? 'percent' : 'px' }; +} diff --git a/scripts/maestro-conformance/pinned-upstream.json b/scripts/maestro-conformance/pinned-upstream.json new file mode 100644 index 000000000..e7668f35d --- /dev/null +++ b/scripts/maestro-conformance/pinned-upstream.json @@ -0,0 +1,25 @@ +{ + "project": "mobile-dev-inc/Maestro", + "version": "2.5.1", + "tag": "v2.5.1", + "commit": "a4c7c95f5ba1884858f7e35efa6b8e0165db9448", + "sourceUrl": "https://github.com/mobile-dev-inc/Maestro/tree/v2.5.1", + "repository": "https://repo1.maven.org/maven2", + "artifacts": [ + { + "coordinate": "dev.mobile:maestro-orchestra:2.5.1", + "role": "Layer 1 YAML parser (YamlCommandReader) and Layer 2 retry/erase constants (Orchestra).", + "sha256": "bbd8e5e35e3c706c6d4760bb1026162474b171155a5fa76f6de1360ab3705468" + }, + { + "coordinate": "dev.mobile:maestro-orchestra-models:2.5.1", + "role": "Command model classes and parser-observed defaults (Layer 2 model defaults).", + "sha256": "0c147f7dff3704e1e04cb728339f2637a6086c234d7984c1434536680f6749ee" + }, + { + "coordinate": "dev.mobile:maestro-client:2.5.1", + "role": "Layer 2 driver constants (Maestro.SCREENSHOT_DIFF_THRESHOLD/ANIMATION_TIMEOUT_MS, IOSDriver.SCREEN_SETTLE_TIMEOUT_MS).", + "sha256": "a6b8c63c0858e8e256752f617fd27b35b0be48e78d367f9cbdffa735a6fcc991" + } + ] +} diff --git a/scripts/maestro-conformance/regenerate.mjs b/scripts/maestro-conformance/regenerate.mjs new file mode 100644 index 000000000..b6fca1786 --- /dev/null +++ b/scripts/maestro-conformance/regenerate.mjs @@ -0,0 +1,116 @@ +#!/usr/bin/env node +// Regenerates the checked-in layer-1 and layer-2 fixtures from the pinned +// upstream Maestro artifacts. This is a manual, toolchain-heavy operation run +// ONLY when the upstream pin changes — normal CI verifies the checked-in +// fixtures deterministically (no Java) via `verify.ts`. +// +// Steps: +// 1. Resolve the pinned dev.mobile jars via the Gradle harness. +// 2. Verify each pinned artifact's SHA-256 against `pinned-upstream.json` +// (the integrity gate the old hand-typed harness never enforced). +// 3. Run the harness over the corpus to emit generated layer-1/layer-2 JSON. +// 4. Wrap each with the upstream pin and write it to `fixtures/`. +// +// Requirements: JDK 17+. Gradle is provided by the committed wrapper +// (`jvm-harness/gradlew`); override with MAESTRO_CONFORMANCE_GRADLE to reuse an +// existing Gradle install. +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { writeManifest } from './build-manifest.mjs'; +import { fixtureContentHash } from './fixture-seal.mjs'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const HARNESS_DIR = path.join(HERE, 'jvm-harness'); +const CORPUS_DIR = path.join(HERE, 'corpus'); +const FIXTURES_DIR = path.join(HERE, 'fixtures'); + +function readPin() { + return JSON.parse(fs.readFileSync(path.join(HERE, 'pinned-upstream.json'), 'utf8')); +} + +function gradle(args, options = {}) { + const override = process.env.MAESTRO_CONFORMANCE_GRADLE; + const [cmd, baseArgs] = override + ? [override, []] + : [process.platform === 'win32' ? 'gradlew.bat' : './gradlew', []]; + return execFileSync(cmd, [...baseArgs, '-p', HARNESS_DIR, '--no-daemon', '--console=plain', '-q', ...args], { + cwd: HARNESS_DIR, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'inherit'], + ...options, + }); +} + +function sha256(filePath) { + return createHash('sha256').update(fs.readFileSync(filePath)).digest('hex'); +} + +function verifyArtifacts(pin) { + const output = gradle(['printUpstreamJars']); + const resolved = new Map(); + for (const line of output.split('\n')) { + const match = /^UPSTREAM_JAR (\S+):(\S+) (.+)$/.exec(line.trim()); + if (match) resolved.set(`dev.mobile:${match[1]}:${match[2]}`, match[3]); + } + for (const artifact of pin.artifacts) { + const jarPath = resolved.get(artifact.coordinate); + if (!jarPath) throw new Error(`Pinned artifact ${artifact.coordinate} was not resolved by Gradle.`); + const actual = sha256(jarPath); + if (actual !== artifact.sha256) { + throw new Error( + `SHA-256 mismatch for ${artifact.coordinate}\n pinned: ${artifact.sha256}\n resolved: ${actual}\n ${jarPath}`, + ); + } + console.log(`verified ${artifact.coordinate} (${actual.slice(0, 12)}…)`); + } +} + +function writeFixture(name, pin, content) { + fs.mkdirSync(FIXTURES_DIR, { recursive: true }); + const { version, tag, commit, project } = pin; + const wrapped = { + schemaVersion: 2, + generatedBy: 'scripts/maestro-conformance/regenerate.mjs', + upstream: { project, version, tag, commit, artifacts: pin.artifacts }, + ...content, + }; + // Seal the generated content. Verifying only the embedded upstream pin would + // let a hand edit to the captured commands/constants pass CI — which is the + // exact transcription failure mode this oracle exists to remove. The seal is + // recomputed on every verify run, so an edit must also forge the hash; the + // scheduled conformance-regenerate job then re-derives from upstream and fails + // on any byte difference, which forgery cannot survive. + const wrappedWithSeal = { ...wrapped, contentHash: fixtureContentHash(wrapped) }; + const target = path.join(FIXTURES_DIR, name); + fs.writeFileSync(target, `${JSON.stringify(wrappedWithSeal, null, 2)}\n`); + console.log(`wrote ${path.relative(HERE, target)}`); +} + +function main() { + const pin = readPin(); + console.log(`Regenerating Maestro ${pin.version} conformance fixtures (${pin.commit.slice(0, 12)}).`); + + // Refresh corpus provenance first so a newly added flow is picked up. + const manifest = writeManifest(pin); + console.log(`corpus manifest: ${manifest.flows.length} flows`); + + verifyArtifacts(pin); + + const outDir = fs.mkdtempSync(path.join(os.tmpdir(), 'maestro-conformance-')); + try { + gradle(['run', `--args=--corpus ${CORPUS_DIR} --out ${outDir}`], { stdio: 'inherit' }); + const layer1 = JSON.parse(fs.readFileSync(path.join(outDir, 'layer1-parser.json'), 'utf8')); + const layer2 = JSON.parse(fs.readFileSync(path.join(outDir, 'layer2-semantics.json'), 'utf8')); + writeFixture('layer1-parser.json', pin, layer1); + writeFixture('layer2-semantics.json', pin, layer2); + } finally { + fs.rmSync(outDir, { recursive: true, force: true }); + } + console.log('Done. Review the diff, then run `node --experimental-strip-types scripts/maestro-conformance/verify.test.ts`.'); +} + +main(); diff --git a/scripts/maestro-conformance/verify.test.ts b/scripts/maestro-conformance/verify.test.ts new file mode 100644 index 000000000..6271f1ebe --- /dev/null +++ b/scripts/maestro-conformance/verify.test.ts @@ -0,0 +1,166 @@ +// Deterministic conformance gate. Runs in normal CI via `node --test` (no Java), +// the same pattern as the layering-guard job. It replays the checked-in, +// JVM-generated fixtures against the live agent-device engine. +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { test } from 'node:test'; +import { + type FlowResult, + checkCoverage, + checkFixtureSeals, + checkLayer2, + classifyAllFlows, + loadLayer1, + loadLayer2, +} from './verify.ts'; +import { + DOCUMENTED_DEVIATIONS, + FLOW_DIVERGENCES, + LAYER2_REFERENCE_ONLY, +} from './expected-divergence.ts'; +// @ts-expect-error -- .mjs helper shared with regenerate.mjs; no type declarations. +import { checkFixtureSeal } from './fixture-seal.mjs'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const PIN = JSON.parse(fs.readFileSync(path.join(HERE, 'pinned-upstream.json'), 'utf8')); + +function flowsById(): Map { + return new Map(classifyAllFlows().map((flow) => [flow.id, flow])); +} + +test('fixture content is sealed against hand editing', () => { + for (const result of checkFixtureSeals()) { + assert.ok(result.actual, `${result.file} has no contentHash — regenerate it`); + assert.equal( + result.actual, + result.expected, + `${result.file} content does not match its seal. Fixtures are generated: run \`pnpm maestro:conformance:regenerate\` rather than editing them by hand.`, + ); + } +}); + +// The seal is only worth having if it actually catches an edit. Prove it does, +// rather than trusting that a hash comparison must work. +test('the seal rejects an edited capture (proof the check has teeth)', () => { + const original = JSON.parse( + fs.readFileSync(path.join(HERE, 'fixtures', 'layer2-semantics.json'), 'utf8'), + ); + // Forge the retry cap the way a hand-transcribed fixture would have drifted. + const tampered = structuredClone(original); + const retryCap = tampered.constants.find((c: { id: string }) => c.id === 'retryMaxRetries'); + retryCap.value = 99; + const { expected, actual } = checkFixtureSeal(tampered); + assert.notEqual(expected, actual, 'editing a captured constant must break the seal'); + // ...and the untouched fixture still verifies, so the check is not just always-fail. + const clean = checkFixtureSeal(original); + assert.equal(clean.expected, clean.actual); +}); + +test('fixtures pin the reviewed upstream Maestro artifacts', () => { + for (const fixture of [loadLayer1(), loadLayer2()] as Array<{ upstream?: unknown }>) { + const upstream = (fixture as { upstream: { version: string; commit: string; artifacts: unknown } }) + .upstream; + assert.equal(upstream.version, PIN.version, 'fixture must pin the reviewed version'); + assert.equal(upstream.commit, PIN.commit, 'fixture must pin the reviewed commit'); + assert.deepEqual(upstream.artifacts, PIN.artifacts, 'fixture jar SHAs must match pinned-upstream.json'); + } +}); + +test('layer 1: our engine never accepts a flow upstream rejects', () => { + const lenient = classifyAllFlows().filter((flow) => flow.classification === 'we-are-lenient'); + assert.deepEqual( + lenient.map((flow) => flow.id), + [], + 'agent-device parsed a flow that upstream Maestro rejects — a conformance regression', + ); +}); + +test('layer 1: every divergence is declared (no silent drift)', () => { + const flows = classifyAllFlows(); + const problems: string[] = []; + for (const flow of flows) { + const declared = FLOW_DIVERGENCES[flow.id]; + if (flow.classification === 'identical' || flow.classification === 'both-reject') { + if (declared) problems.push(`${flow.id}: declared divergence but classified ${flow.classification}`); + continue; + } + if (!declared) { + problems.push(`${flow.id}: undeclared ${flow.classification}${flow.detail ? `\n ${flow.detail}` : ''}`); + continue; + } + if (declared.classification !== flow.classification) { + problems.push(`${flow.id}: declared ${declared.classification} but classified ${flow.classification}`); + } + if (flow.classification === 'we-reject' && !(declared.unsupported && declared.unsupported.length > 0)) { + problems.push(`${flow.id}: we-reject entries must list the unsupported command(s)/option(s)`); + } + } + assert.deepEqual(problems, [], `Undeclared or mismatched divergences:\n ${problems.join('\n ')}`); +}); + +test('layer 1: no stale divergence declarations', () => { + const ids = new Set(classifyAllFlows().map((flow) => flow.id)); + const stale = Object.keys(FLOW_DIVERGENCES).filter((id) => !ids.has(id)); + assert.deepEqual(stale, [], 'FLOW_DIVERGENCES references flows no longer in the corpus'); +}); + +test('layer 2: generated semantic vectors match live engine constants', () => { + const results = checkLayer2(); + const mismatched = results.filter((result) => result.status === 'mismatch'); + assert.deepEqual( + mismatched.map((result) => `${result.id}: upstream=${result.upstream} agent=${result.agent}`), + [], + 'a layer-2 semantic vector drifted from its agent-device constant', + ); + // Every reference-only vector must be an on-the-record deviation. + for (const result of results) { + if (result.status === 'reference-only') { + const documented = DOCUMENTED_DEVIATIONS.some((d) => d.description.includes(result.id) || LAYER2_REFERENCE_ONLY.has(result.id)); + assert.ok(documented, `reference-only vector ${result.id} must be a documented deviation`); + } + } +}); + +test('coverage: every supported command is fixture-backed or explicitly unverified', () => { + const gaps = checkCoverage().filter((result) => !result.covered && !result.unverified); + assert.deepEqual( + gaps.map((result) => result.command), + [], + 'supported commands with no corpus coverage — add a flow or list them in UNVERIFIED_COMMANDS', + ); +}); + +// --- The four #1217 bug classes, each tied to its fixture --- + +test('bug class 1: decimal percentage coordinates are rejected (not rounded)', () => { + const flow = flowsById().get('bug-classes/percent-decimal-swipe'); + assert.equal(flow?.upstreamStatus, 'rejected', 'upstream rejects "50.5%, 50%"'); + assert.equal(flow?.classification, 'both-reject', 'agent-device must also reject decimals'); +}); + +test('bug class 2: a target swipe without a direction is rejected', () => { + const flow = flowsById().get('bug-classes/target-swipe-missing-direction'); + assert.equal(flow?.upstreamStatus, 'rejected', 'upstream requires an explicit direction'); + assert.equal(flow?.classification, 'both-reject', 'agent-device must also reject it'); +}); + +test('bug class 3: the retry cap matches the upstream MAX_RETRIES_ALLOWED constant', () => { + const retryCap = checkLayer2().find((result) => result.id === 'retryMaxRetries'); + assert.equal(retryCap?.status, 'match'); + assert.equal(retryCap?.upstream, 3, 'upstream clamps retry blocks to 3'); + // Parse parity: an over-cap maxRetries is stored verbatim (the clamp is runtime). + assert.equal(flowsById().get('bug-classes/retry-over-cap')?.classification, 'identical'); +}); + +test('bug class 4: settle default parses identically; ordering is a layer-3 differential', () => { + assert.equal(flowsById().get('bug-classes/settle-after-tap')?.classification, 'identical'); + // The 200ms x 10 settle loop has no reflectable upstream constant; the + // sleep-after-capture ordering is verified by the layer-3 differential scenario. + const layer2 = loadLayer2(); + assert.ok( + !layer2.constants.some((constant) => /settle/i.test(constant.id) && constant.id !== 'iosScreenSettleTimeoutMs'), + 'no upstream settle-loop constant exists to cross-check; keep this as layer 3', + ); +}); diff --git a/scripts/maestro-conformance/verify.ts b/scripts/maestro-conformance/verify.ts new file mode 100644 index 000000000..edaae87f8 --- /dev/null +++ b/scripts/maestro-conformance/verify.ts @@ -0,0 +1,259 @@ +// Deterministic Maestro conformance verifier. Runs in normal CI with no Java: +// it replays the checked-in, JVM-generated fixtures against our live engine. +// +// - Layer 1: parse every corpus flow with our engine, classify it against the +// upstream parser capture (identical / both-reject / we-reject / mismatch / +// we-are-lenient), and require every non-identical outcome to be a declared +// expected divergence. +// - Layer 2: cross-check each generated semantic vector against the live +// agent-device constant it mirrors. +// - Coverage: every command in the support matrix must be exercised by a corpus +// flow our engine parses, or be explicitly listed as unverified. +// - Bug classes: the four #1217 regressions each assert against their fixture. +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { AppError } from '../../src/kernel/errors.ts'; +import type { MaestroProgram } from '../../src/compat/maestro/program-ir.ts'; +import { parseMaestroProgram } from '../../src/compat/maestro/program-ir-parser.ts'; +import { SUPPORTED_MAESTRO_COMMAND_NAMES } from '../../src/compat/maestro/program-ir-command-parser.ts'; +import { MAESTRO_COMPATIBILITY_PRESETS } from '../../src/compat/maestro/compatibility-policy.ts'; +import { + type CanonicalCommand, + canonicalizeAgentCommands, + canonicalizeUpstreamFlow, +} from './normalize.ts'; +import { LAYER2_REFERENCE_ONLY, UNVERIFIED_COMMANDS } from './expected-divergence.ts'; +// @ts-expect-error -- .mjs helper shared with regenerate.mjs; no type declarations. +import { checkFixtureSeal } from './fixture-seal.mjs'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +export const CORPUS_DIR = path.join(HERE, 'corpus'); +const FIXTURES_DIR = path.join(HERE, 'fixtures'); + +export type Classification = + | 'identical' + | 'both-reject' + | 'we-reject' + | 'mismatch' + | 'we-are-lenient'; + +export type FlowResult = { + id: string; + file: string; + upstreamStatus: 'parsed' | 'rejected'; + agentStatus: 'parsed' | 'rejected'; + classification: Classification; + detail?: string; +}; + +type Layer1Fixture = { + flows: Array<{ + id: string; + file: string; + status: 'parsed' | 'rejected'; + commands?: Array<{ type: string; fields: Record }>; + error?: { class: string; message: string }; + }>; +}; + +type Layer2Fixture = { + constants: Array<{ id: string; symbol: string; value: number }>; + modelDefaults: Array<{ id: string; value: number }>; +}; + +function readJson(file: string): T { + return JSON.parse(fs.readFileSync(file, 'utf8')) as T; +} + +export const FIXTURE_FILES = ['layer1-parser.json', 'layer2-semantics.json'] as const; + +export function loadLayer1(): Layer1Fixture { + return readJson(path.join(FIXTURES_DIR, 'layer1-parser.json')); +} + +export function loadLayer2(): Layer2Fixture { + return readJson(path.join(FIXTURES_DIR, 'layer2-semantics.json')); +} + +export type SealResult = { file: string; sealed: boolean; expected: string; actual?: string }; + +/** + * Recompute each fixture's content seal. This is what makes "generated from + * upstream" an enforced property rather than a claim in a README: editing a + * captured command or constant by hand changes the content and fails here. + */ +export function checkFixtureSeals(): SealResult[] { + return FIXTURE_FILES.map((file) => { + const parsed = readJson>(path.join(FIXTURES_DIR, file)); + const { expected, actual } = checkFixtureSeal(parsed); + return { file, sealed: expected === actual, expected, actual: actual as string | undefined }; + }); +} + +/** + * Error codes that count as a deliberate parser rejection. Every rejection the + * Maestro parser raises — unsupported command/option, bad value, and even a YAML + * syntax error — is wrapped as AppError('INVALID_ARGS'). Anything else (a + * TypeError, a bug in our own parser) is a crash, not a rejection: rethrow it so + * it surfaces loudly instead of being laundered into a `we-reject` that a + * declared divergence would then silently accept. + */ +const AGENT_REJECTION_CODES = new Set(['INVALID_ARGS']); + +/** Parse a corpus flow with the live engine. `null` = a clean rejection. */ +function agentParseProgram(file: string): MaestroProgram | null { + const script = fs.readFileSync(path.join(CORPUS_DIR, file), 'utf8'); + try { + return parseMaestroProgram(script, { sourcePath: file }); + } catch (error) { + if (error instanceof AppError && AGENT_REJECTION_CODES.has(error.code)) return null; + throw new Error(`Parsing ${file} crashed instead of rejecting cleanly`, { cause: error }); + } +} + +function agentParse(file: string): { status: 'parsed' | 'rejected'; commands?: CanonicalCommand[] } { + const program = agentParseProgram(file); + if (!program) return { status: 'rejected' }; + return { status: 'parsed', commands: canonicalizeAgentCommands(program) }; +} + +function classifyFlow(fixtureFlow: Layer1Fixture['flows'][number]): FlowResult { + const agent = agentParse(fixtureFlow.file); + const upstreamStatus = fixtureFlow.status; + const base = { id: fixtureFlow.id, file: fixtureFlow.file, upstreamStatus, agentStatus: agent.status }; + + if (upstreamStatus === 'rejected') { + return { + ...base, + classification: agent.status === 'rejected' ? 'both-reject' : 'we-are-lenient', + }; + } + // upstream parsed + if (agent.status === 'rejected') { + return { ...base, classification: 'we-reject' }; + } + const upstream = canonicalizeUpstreamFlow(fixtureFlow.commands ?? []); + const agentCommands = agent.commands ?? []; + const upstreamJson = JSON.stringify(upstream); + const agentJson = JSON.stringify(agentCommands); + if (upstreamJson === agentJson) return { ...base, classification: 'identical' }; + return { + ...base, + classification: 'mismatch', + detail: `upstream=${upstreamJson}\n agent =${agentJson}`, + }; +} + +export function classifyAllFlows(): FlowResult[] { + return loadLayer1().flows.map(classifyFlow); +} + +// --------------------------------------------------------------------------- +// Layer 2 cross-check +// --------------------------------------------------------------------------- + +const P = MAESTRO_COMPATIBILITY_PRESETS; + +/** Generated vector id → the live agent-device constant it must equal. */ +const LAYER2_AGENT_CONSTANTS: Record = { + retryMaxRetries: P.control.retryMaxRetries, + animationWaitThreshold: P.command.waitForAnimationToEndDifferencePercent, + animationWaitTimeoutMs: P.command.waitForAnimationToEndTimeoutMs, + maxEraseCharacters: P.command.eraseTextMaxCharacters, + swipeDurationMs: P.command.swipeDurationMs, +}; + +export type Layer2Result = { + id: string; + upstream: number; + agent?: number; + status: 'match' | 'mismatch' | 'reference-only'; +}; + +export function checkLayer2(): Layer2Result[] { + const fixture = loadLayer2(); + const vectors = [...fixture.constants, ...fixture.modelDefaults]; + return vectors.map((vector) => { + if (LAYER2_REFERENCE_ONLY.has(vector.id)) { + return { id: vector.id, upstream: vector.value, status: 'reference-only' as const }; + } + const agent = LAYER2_AGENT_CONSTANTS[vector.id]; + if (agent === undefined) { + return { id: vector.id, upstream: vector.value, status: 'reference-only' as const }; + } + return { + id: vector.id, + upstream: vector.value, + agent, + status: agent === vector.value ? ('match' as const) : ('mismatch' as const), + }; + }); +} + +// --------------------------------------------------------------------------- +// Support-matrix coverage +// --------------------------------------------------------------------------- + +export type CoverageResult = { command: string; covered: boolean; unverified: boolean }; + +/** Which native agent-device command kinds each corpus flow parses to. */ +export function agentKindsByCorpus(): Set { + const kinds = new Set(); + for (const flow of loadLayer1().flows) { + // Rejected flows contribute nothing to coverage; a crash still throws. + const program = agentParseProgram(flow.file); + if (program) collectKinds(program.commands, kinds); + } + return kinds; +} + +function collectKinds(commands: Array<{ kind: string; commands?: unknown }>, into: Set): void { + for (const command of commands) { + into.add(command.kind); + const nested = (command as { commands?: Array<{ kind: string }> }).commands; + if (Array.isArray(nested)) collectKinds(nested, into); + } +} + +export function checkCoverage(): CoverageResult[] { + const covered = agentKindsByCorpus(); + return SUPPORTED_MAESTRO_COMMAND_NAMES.map((command) => ({ + command, + covered: covered.has(command), + unverified: UNVERIFIED_COMMANDS.has(command), + })); +} + +// --------------------------------------------------------------------------- +// Report CLI (dev aid; the enforcing checks live in verify.test.ts) +// --------------------------------------------------------------------------- + +function report(): void { + const flows = classifyAllFlows(); + const byClass = new Map(); + for (const flow of flows) { + const group = byClass.get(flow.classification) ?? []; + group.push(flow); + byClass.set(flow.classification, group); + } + for (const [classification, group] of byClass) { + console.log(`\n### ${classification} (${group.length})`); + for (const flow of group) { + console.log(` ${flow.id}`); + if (flow.detail) console.log(` ${flow.detail}`); + } + } + console.log('\n### layer 2'); + for (const result of checkLayer2()) { + console.log(` ${result.status.padEnd(14)} ${result.id} upstream=${result.upstream} agent=${result.agent ?? '-'}`); + } + console.log('\n### coverage gaps'); + for (const result of checkCoverage()) { + if (!result.covered && !result.unverified) console.log(` UNCOVERED ${result.command}`); + } +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + report(); +} diff --git a/src/compat/maestro/__tests__/runtime-port-geometry.test.ts b/src/compat/maestro/__tests__/runtime-port-geometry.test.ts new file mode 100644 index 000000000..d542aaf30 --- /dev/null +++ b/src/compat/maestro/__tests__/runtime-port-geometry.test.ts @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict'; +import { describe, test } from 'vitest'; +import { resolveMaestroCoordinate } from '../runtime-port-geometry.ts'; + +// Bug class 1 (#1217), runtime half: upstream Maestro converts percentage +// coordinates to pixels by integer division (Maestro.kt), so the result is +// TRUNCATED, never rounded. The parse-level half (rejecting decimal percentages) +// is covered by the conformance oracle's layer-1 corpus. +// +// This is deliberately a unit test rather than a layer-3 device scenario: +// truncation and rounding differ by at most one pixel, which no app-observable +// outcome on a real device can distinguish. A pure test of the conversion pins it +// exactly. `resolveMaestroCoordinate` short-circuits on a known viewport, so no +// port or device is involved. +describe('resolveMaestroCoordinate percentage conversion', () => { + const viewport = (width: number, height: number, x = 0, y = 0) => ({ x, y, width, height }); + const resolve = (percent: { x: number; y: number }, vp: ReturnType) => + resolveMaestroCoordinate( + { space: 'percent', x: percent.x, y: percent.y }, + undefined as never, + undefined as never, + vp, + ); + + test('truncates rather than rounds when the pixel is fractional', async () => { + // 1125 * 50 / 100 = 562.5 -> trunc 562 (rounding would give 563) + // 2436 * 33 / 100 = 803.88 -> trunc 803 (rounding would give 804) + const point = await resolve({ x: 50, y: 33 }, viewport(1125, 2436)); + assert.deepEqual(point, { x: 562, y: 803 }); + }); + + test('truncates fractions above .5, where rounding would go up', async () => { + // Both fractions round UP, so these only pass under truncation: + // 1179 * 5 / 100 = 58.95 -> trunc 58 (round -> 59) + // 2556 * 35 / 100 = 894.6 -> trunc 894 (round -> 895) + const point = await resolve({ x: 5, y: 35 }, viewport(1179, 2556)); + assert.deepEqual(point, { x: 58, y: 894 }); + }); + + test('is exact when the pixel is whole', async () => { + const point = await resolve({ x: 50, y: 25 }, viewport(1080, 1920)); + assert.deepEqual(point, { x: 540, y: 480 }); + }); + + test('adds the viewport origin after truncating the fraction', async () => { + // Origin is added to the truncated span, not truncated together with it. + const point = await resolve({ x: 50, y: 50 }, viewport(1125, 2436, 7, 11)); + assert.deepEqual(point, { x: 7 + 562, y: 11 + 1218 }); + }); + + test('0% and 100% map to the viewport edges', async () => { + assert.deepEqual(await resolve({ x: 0, y: 0 }, viewport(1125, 2436)), { x: 0, y: 0 }); + assert.deepEqual(await resolve({ x: 100, y: 100 }, viewport(1125, 2436)), { x: 1125, y: 2436 }); + }); + + test('absolute coordinates pass through untouched', async () => { + const point = await resolveMaestroCoordinate( + { space: 'absolute', x: 100, y: 200 }, + undefined as never, + undefined as never, + viewport(1125, 2436), + ); + assert.deepEqual(point, { x: 100, y: 200 }); + }); +}); diff --git a/src/compat/maestro/compatibility-policy.ts b/src/compat/maestro/compatibility-policy.ts index 65d011d7a..f95ae50b1 100644 --- a/src/compat/maestro/compatibility-policy.ts +++ b/src/compat/maestro/compatibility-policy.ts @@ -5,8 +5,9 @@ export type MaestroCompatibilityTimingPolicy = { runFlowConditionTimeoutMs: number; }; -// Maestro 2.5.1 defaults at a4c7c95f; pinned source metadata lives in -// scripts/maestro-conformance-fixtures/upstream-maestro-2.5.1.json. +// Maestro 2.5.1 defaults at a4c7c95f. The conformance oracle cross-checks the +// retry cap, swipe duration, erase cap, and animation-wait constants below +// against JVM-generated semantic vectors (scripts/maestro-conformance). export const MAESTRO_COMPATIBILITY_PRESETS = { control: { retryMaxRetries: 3, diff --git a/src/compat/maestro/program-ir-command-parser.ts b/src/compat/maestro/program-ir-command-parser.ts index b85921b5e..f132decce 100644 --- a/src/compat/maestro/program-ir-command-parser.ts +++ b/src/compat/maestro/program-ir-command-parser.ts @@ -125,6 +125,16 @@ const COMMAND_VALUE_PARSERS: Readonly> = { parseMaestroRetryCommand(value, node, context, parseMaestroCommandList), }; +/** + * The exact set of Maestro command names our engine accepts. This is the + * authoritative supported surface — any name outside it is rejected by + * `parseCommandValue`. The conformance oracle + * (`scripts/maestro-conformance/verify.ts`) asserts every entry is either + * corpus-covered or explicitly listed as unverified. + */ +export const SUPPORTED_MAESTRO_COMMAND_NAMES: readonly string[] = + Object.keys(COMMAND_VALUE_PARSERS); + function parseCommandValue( name: string, value: Node | null, diff --git a/vitest.config.ts b/vitest.config.ts index f97700b87..b3eab91b7 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -24,7 +24,8 @@ export default defineConfig({ include: [ 'src/**/*.test.ts', 'scripts/__tests__/help-conformance-bench.test.ts', - 'scripts/maestro-conformance.test.ts', + // The Maestro conformance oracle runs via `node --test` in its own CI + // job (scripts/maestro-conformance), like the layering guard. ], exclude: [ANDROID_ADB_STUB_TESTS], setupFiles: ['src/__tests__/process-memo-setup.ts'],