diff --git a/.github/workflows/live-canary.yml b/.github/workflows/live-canary.yml new file mode 100644 index 000000000..03cd6cf08 --- /dev/null +++ b/.github/workflows/live-canary.yml @@ -0,0 +1,142 @@ +name: Live Canary + +# Runs the BC4↔BC5 live compatibility canary against real Basecamp backends. +# +# Triggers +# -------- +# - workflow_dispatch: manual run, useful for ad-hoc verification. +# - schedule: nightly cron at 09:00 UTC. Catches server-side drift between +# the BC4 (master) baseline and the BC5 (five) branch over time. +# +# Required secrets (configured at repo level) +# ------------------------------------------- +# BASECAMP_TOKEN OAuth token for both backends. Must have read scope +# on the canary's read-only fixture set. +# BASECAMP_ACCOUNT_ID The numeric account ID used to compose the per-backend +# base URL. Same on both backends — the canary requires +# identical account state across BC4 and BC5 for +# pairwise rules to be meaningful. +# BC5_HOST Origin of the BC5 backend (e.g. https://5.basecampapi.com). +# +# Optional secret (defaults) +# -------------------------- +# BASECAMP_HOST Origin of the BC4 backend. Defaults to +# https://3.basecampapi.com when unset. +# +# The job no-ops with a clear log message if BASECAMP_TOKEN is unset (e.g. +# when running from a fork without secrets). It does not fail the run — +# missing secrets are operator-controllable state. +on: + workflow_dispatch: + schedule: + - cron: '0 9 * * *' + +permissions: {} + +concurrency: + group: live-canary + cancel-in-progress: false + +jobs: + canary: + name: BC4↔BC5 canary + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 30 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Check for canary secrets + id: secrets + env: + BASECAMP_TOKEN: ${{ secrets.BASECAMP_TOKEN }} + BASECAMP_ACCOUNT_ID: ${{ secrets.BASECAMP_ACCOUNT_ID }} + BC5_HOST: ${{ secrets.BC5_HOST }} + run: | + if [ -z "$BASECAMP_TOKEN" ] || [ -z "$BASECAMP_ACCOUNT_ID" ] || [ -z "$BC5_HOST" ]; then + echo "Canary secrets not configured; skipping live run." + echo "Required: BASECAMP_TOKEN, BASECAMP_ACCOUNT_ID, BC5_HOST." + echo "configured=false" >> "$GITHUB_OUTPUT" + else + echo "configured=true" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Go + if: steps.secrets.outputs.configured == 'true' + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: 'go/go.mod' + cache-dependency-path: 'go/go.sum' + + - name: Set up Node.js + if: steps.secrets.outputs.configured == 'true' + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: | + typescript/package-lock.json + conformance/runner/typescript/package-lock.json + + - name: Set up Java + if: steps.secrets.outputs.configured == 'true' + uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + if: steps.secrets.outputs.configured == 'true' + uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 + with: + cache-provider: basic + + - name: Set up Ruby + if: steps.secrets.outputs.configured == 'true' + uses: ruby/setup-ruby@d45b1a4e94b71acab930e56e79c6aa188764e7f9 # v1.316.0 + with: + ruby-version: '3.3' + bundler-cache: true + working-directory: conformance/runner/ruby + + - name: Install uv (Python) + if: steps.secrets.outputs.configured == 'true' + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + + - name: Build TypeScript SDK + if: steps.secrets.outputs.configured == 'true' + working-directory: typescript + run: | + npm ci + npm run build + + - name: Run live canary against BC4 + BC5 with pairwise comparison + if: steps.secrets.outputs.configured == 'true' + env: + BASECAMP_TOKEN: ${{ secrets.BASECAMP_TOKEN }} + BASECAMP_ACCOUNT_ID: ${{ secrets.BASECAMP_ACCOUNT_ID }} + # Default the BC4 origin here: an unset optional secret expands to ''. + # The Makefile's recipe-shell ${BASECAMP_HOST:-...} default would also + # catch the empty string, but defaulting explicitly keeps the workflow + # self-describing about which BC4 origin a run used. + BASECAMP_HOST: ${{ secrets.BASECAMP_HOST || 'https://3.basecampapi.com' }} + BC5_HOST: ${{ secrets.BC5_HOST }} + LIVE_RECORD_DIR: tmp/live-canary + run: make check-bc5-compat + + # Deliberately EXCLUDES the wire/ trees: wire snapshots carry raw + # response bodies and captured headers from a live account, which must + # not persist as downloadable workflow artifacts. Decode results carry + # only booleans, error strings, and dotted field paths. + - name: Upload canary decode results + if: always() && steps.secrets.outputs.configured == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: live-canary-${{ github.run_id }} + path: | + tmp/live-canary/*/decode/ + retention-days: 14 + if-no-files-found: warn diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9481719dd..717f914d0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,6 +15,10 @@ Thank you for your interest in contributing to the Basecamp SDK. This document p | Kotlin | JDK 17+, Kotlin 2.0+ | | Python | Python 3.11+, [uv](https://docs.astral.sh/uv/) | +Shared tooling: `jq`, and bash >= 4.4 on `PATH` for the pairwise-canary +scripts that `make check` runs (macOS ships bash 3.2 at `/bin/bash` — +`brew install bash`; the scripts fail fast with this exact hint). + A Basecamp account is optional (for integration testing only). ### Getting Started @@ -275,8 +279,9 @@ Optional env: - `BASECAMP_BACKEND=bc4|bc5` — namespaces persisted snapshots so BC4 and BC5 runs don't collide. - `LIVE_RECORD_DIR=` — persists wire snapshots to - `//wire/.json`. Used by downstream cross-language - decoders (PR 3) and BC4↔BC5 comparison (PR 4). + `//wire/.json`. Consumed by the cross-language + replay runners (`make conformance-*-replay`) and by the pairwise + BC4↔BC5 comparison (`scripts/compare-canary-runs.sh`). - `BASECAMP_BC4_PROJECT_ID` / `BASECAMP_BC5_PROJECT_ID` / `BASECAMP_PROJECT_ID` etc. — explicit fixture-IDs override the runner's discovery walk. Same pattern applies for `TODOSET_ID`, `TODOLIST_ID`, @@ -348,11 +353,6 @@ hand-rolled schema walkers (which mirror the TS validator's required + extras algorithm in each language). Any per-language divergence in `extras_seen` points at a walker bug in the diverging language. -`make conformance-typescript-live` is owned by PR 2; the four -`make conformance-*-replay` targets ship in PR 3. The orchestrating -`make conformance-live` and `make check-bc5-compat` targets that thread -TS capture → replay → BC4↔BC5 comparison together land in PR 4. - When the SDK gains a new operation in `live-my-surface.json`, it must be added to: @@ -364,6 +364,123 @@ added to: Each runner's coverage gate refuses to start until all five are in place. +### Pairwise BC4↔BC5 comparison + +Per-backend schema validation is necessary but not sufficient. With every +new BC5 field marked optional, a regression where BC4 emits `memories: +[item, item]` and BC5 emits `memories: []` would pass per-backend schema +checks — yet that's exactly the additive-only invariant the canary should +catch. The pairwise comparison closes that loop. + +Each live test in `conformance/tests/live-my-surface.json` can carry +`pairwiseAssertions`, which apply to a matched pair of BC4 and BC5 wire +snapshots for the same operation. Four rule types: + +- `pairwiseSupersetArray` — BC5 array length at each path must be ≥ BC4's. + Catches "this array shrank between backends". +- `pairwiseSupersetKeys` — BC5 object's keys at each path must be a superset + of BC4's keys. Catches "this field disappeared". +- `pairwiseEqual` — BC5 value at each path must equal BC4's. Use sparingly; + most useful for type-discriminator fields. +- `pairwiseDeltaAllowed` — paths where divergence is explicitly accepted. + The listed paths are skipped by the other three rules for this operation. + `reason` is **required** — a public compatibility report indexes accepted + divergences. + +Path syntax (dotted identifiers, evaluated against each snapshot): + +- `""` (empty string) addresses the body root. +- `foo.bar` is shorthand for `pages[0].body.foo.bar` — single-page bodies. +- `pages[N].body.X` addresses a specific page in multi-page snapshots. +- `pages[*].body.X` aggregates across pages into a list (each page's value + becomes one element of the result). `pairwiseEqual` compares that list as + a single value. `pairwiseSupersetArray` compares the **total item count + across pages** (a page missing the field contributes 0; any non-null + non-array page value is invalid), so a field dropped on every page fails + even when page counts match, and redistribution across page boundaries + with the total preserved passes. `pairwiseSupersetKeys` does **not** + support `pages[*]` — the aggregate is a list, not an object, and the rule + flags it as an invalid target; address a specific page + (`pages[N].body.X`) for keys checks on paginated endpoints. + +Example — the canonical `memories` canary on `GetMyNotifications`: + +```json +"pairwiseAssertions": [ + { + "type": "pairwiseSupersetArray", + "paths": ["memories"], + "reason": "Additive-only invariant: BC5 memories[] should remain a superset of BC4's. PERMANENTLY waived by the pairwiseDeltaAllowed entry below — bc3 documents memories as an always-empty placeholder on BC5 (doc/api/sections/my_notifications.md). See spec/api-gaps/memories-emptied-regression.md." + }, + { + "type": "pairwiseDeltaAllowed", + "paths": ["memories"], + "reason": "PERMANENT documented contract divergence: BC5 ships `json.memories []` by contract — doc/api/sections/my_notifications.md (bc3 #11628) codifies memories as an always-empty placeholder superseded by bubble_ups. The once-planned alias (`json.memories @bubble_ups`, bc3 #10947) never shipped; #10947 is closed unmerged. Not a pending regression — this entry is the machine-readable record of an accepted BC4→BC5 subtractive delta. Retire only if BC4 (the four branch) empties memories too (the superset rule then passes clean) or BC5's documented contract changes. Tracked in spec/api-gaps/memories-emptied-regression.md." + } +] +``` + +BC5 ships `memories: []` permanently by documented contract — +`doc/api/sections/my_notifications.md` (bc3 #11628) codifies memories as an +always-empty placeholder superseded by `bubble_ups`. The superset rule stays +as the encoded additive-only invariant; the `pairwiseDeltaAllowed` entry is +the machine-readable record of the accepted delta, feeding the public +compatibility report. Retire the waiver only if BC4 (the four branch) empties +memories too — the superset rule then passes clean — or BC5's documented +contract changes. + +### Orchestrator + +`make check-bc5-compat` threads the two-backend capture plus pairwise +comparison together: + +```bash +BASECAMP_TOKEN= \ +BASECAMP_ACCOUNT_ID= \ +BC5_HOST=https://5.basecampapi.com \ +make check-bc5-compat +``` + +What it runs, in order: + +1. `BASECAMP_BACKEND=bc4 LIVE_RECORD_DIR=tmp/live-canary make conformance-live` + — TS captures wire snapshots from BC4 (defaulting `BASECAMP_HOST` to + `https://3.basecampapi.com`), then Ruby/Python/Go/Kotlin replay-decode. +2. `BASECAMP_BACKEND=bc5 LIVE_RECORD_DIR=tmp/live-canary make conformance-live` + — same against the BC5 origin set via `BC5_HOST`. +3. `./scripts/compare-canary-runs.sh tmp/live-canary/bc4/wire tmp/live-canary/bc5/wire` + — applies pairwise rules. Reports all violations outside + `pairwiseDeltaAllowed` and exits non-zero if any are found. + +Override `LIVE_RECORD_DIR` (default `tmp/live-canary`) or `BASECAMP_HOST` +(default `https://3.basecampapi.com`) on the make line. + +**Identical account state across both runs is mandatory.** The pairwise +rules assume the BC4 and BC5 backends see the same projects, the same +todosets, the same reading list, etc. Without that, `unreads`/`reads` +arrays drift naturally between captures and pairwise asserts will false-fail. +Use a dedicated test account with stable, equivalent fixtures (e.g. a +sandbox account snapshot replicated to each backend). + +### Scheduled CI + +`.github/workflows/live-canary.yml` runs `check-bc5-compat` nightly via +cron and on `workflow_dispatch`. It is **opt-in**: the workflow no-ops +with a clear log message if the required repo secrets aren't configured. + +Required secrets: + +- `BASECAMP_TOKEN` — OAuth token with read scope for the canary fixtures. +- `BASECAMP_ACCOUNT_ID` — the numeric account ID used on both backends. +- `BC5_HOST` — origin of the BC5 backend. + +Optional: + +- `BASECAMP_HOST` — origin of the BC4 backend; defaults to `https://3.basecampapi.com`. + +Snapshots are uploaded as a workflow artifact (`live-canary-`, +14-day retention) so failures can be inspected post-hoc without rerunning. + ## API gap registry (`spec/api-gaps/`) When BC ships a new user-visible feature without a JSON API (or with an diff --git a/COORDINATION.md b/COORDINATION.md index 7e46abad2..bde908154 100644 --- a/COORDINATION.md +++ b/COORDINATION.md @@ -68,7 +68,6 @@ Both records live on in the registry and this SDK plan's §8. `five+api`) - SDK-side absorption status: [`spec/api-gaps/README.md`](spec/api-gaps/README.md) - Live canary results: `make check-bc5-compat` against current refs - (lands in PR 4 of the SDK plan) - Cross-team alignment status: this file — The basecamp-sdk team diff --git a/Makefile b/Makefile index 126b8f29f..16abd1b21 100644 --- a/Makefile +++ b/Makefile @@ -384,7 +384,7 @@ py-clean: # Conformance Test targets #------------------------------------------------------------------------------ -.PHONY: conformance conformance-go conformance-go-replay conformance-kotlin conformance-kotlin-replay conformance-typescript conformance-typescript-live conformance-ruby conformance-ruby-replay conformance-python conformance-python-replay conformance-build oauth-fixtures-check +.PHONY: conformance conformance-go conformance-go-replay conformance-kotlin conformance-kotlin-replay conformance-typescript conformance-typescript-live conformance-ruby conformance-ruby-replay conformance-python conformance-python-replay conformance-build conformance-live check-bc5-compat oauth-fixtures-check # Pinned validator for the data-only OAuth discovery fixtures. Run via uvx so the # version is reproducible without a global install; the schema is separate from @@ -474,6 +474,112 @@ conformance-python-replay: conformance: oauth-fixtures-check conformance-go conformance-kotlin conformance-typescript conformance-ruby conformance-python @echo "==> Conformance tests passed" +# Orchestrate one canary pass against a single backend: +# 1. TS captures canonical wire snapshots (live HTTP). +# 2. Each replay runner decodes those snapshots through its SDK + walks raw JSON. +# +# Required env (passed through to children): +# BASECAMP_TOKEN, BASECAMP_ACCOUNT_ID, BASECAMP_BACKEND, LIVE_RECORD_DIR +# (BASECAMP_LIVE=1 is set by the conformance-typescript-live recipe itself.) +# +# The four replay runners run sequentially after the TS capture completes +# (the per-language replays need the wire snapshots TS just wrote). They +# read from `$$LIVE_RECORD_DIR/$$BASECAMP_BACKEND/wire/` and write to +# `$$LIVE_RECORD_DIR/$$BASECAMP_BACKEND/decode//`. Failures in any +# stage fail the orchestrator. +# +# Opt-in target: not invoked by `make check`. +conformance-live: + @test -n "$$LIVE_RECORD_DIR" || (echo "LIVE_RECORD_DIR is required" >&2; exit 1) + @test -n "$$BASECAMP_BACKEND" || (echo "BASECAMP_BACKEND is required" >&2; exit 1) + @test -n "$$BASECAMP_TOKEN" || (echo "BASECAMP_TOKEN is required" >&2; exit 1) + @test -n "$$BASECAMP_ACCOUNT_ID" || (echo "BASECAMP_ACCOUNT_ID is required" >&2; exit 1) + @# Canonicalize LIVE_RECORD_DIR before fanning out: the capture and + @# replay recipes each `cd` into their runner directory, so a relative + @# path (e.g. the documented tmp/live-canary) would scatter snapshots + @# under conformance/runner/*/ while the pairwise compare reads from + @# the repo root — missing-snapshot errors instead of a comparison. + @LRD="$$LIVE_RECORD_DIR"; case "$$LRD" in /*) ;; *) LRD="$$(pwd)/$$LRD" ;; esac; \ + echo "==> conformance-live: capturing canonical wire snapshots (TypeScript)..." && \ + LIVE_RECORD_DIR="$$LRD" $(MAKE) conformance-typescript-live && \ + echo "==> Running cross-language wire-replay against just-captured snapshots..." && \ + WIRE_REPLAY_DIR="$$LRD" $(MAKE) conformance-ruby-replay && \ + WIRE_REPLAY_DIR="$$LRD" $(MAKE) conformance-python-replay && \ + WIRE_REPLAY_DIR="$$LRD" $(MAKE) conformance-go-replay && \ + WIRE_REPLAY_DIR="$$LRD" $(MAKE) conformance-kotlin-replay + @echo "==> conformance-live: capture + replay complete for backend $$BASECAMP_BACKEND" + +# Top-level orchestrator: full canary against BC4 then BC5, then pairwise comparison. +# +# Required env: +# BASECAMP_TOKEN, BASECAMP_ACCOUNT_ID, BC5_HOST (BC5 backend origin) +# Optional env: +# BASECAMP_HOST (BC4 backend origin; defaults to https://3.basecampapi.com) +# LIVE_RECORD_DIR (snapshot root; defaults to tmp/live-canary) +# +# Snapshot dirs are per-backend: $$LIVE_RECORD_DIR/{bc4,bc5}/{wire,decode}/. +# The TS runner writes to `$$LIVE_RECORD_DIR/$$BASECAMP_BACKEND/wire/`, so the +# orchestrator distinguishes runs by switching BASECAMP_BACKEND and the host +# while reusing one snapshot root. +# +# Each pass must pass per-backend (TS schema validation + 4-language decode). +# After both, the pairwise script applies the additive-only invariant to BC4 +# vs BC5 snapshots, reports all violations outside pairwiseDeltaAllowed, and +# exits non-zero if any are found. +# +# Account state must be identical across the two runs — see CONTRIBUTING.md. +check-bc5-compat: + @test -n "$$BASECAMP_TOKEN" || (echo "BASECAMP_TOKEN is required" >&2; exit 2) + @test -n "$$BASECAMP_ACCOUNT_ID" || (echo "BASECAMP_ACCOUNT_ID is required" >&2; exit 2) + @test -n "$$BC5_HOST" || (echo "BC5_HOST is required (BC5 backend origin, e.g. https://5.basecampapi.com)" >&2; exit 2) + @# Defaults resolve in the recipe shell, NOT via target-specific `?=`: + @# make 3.81 (macOS /usr/bin/make) leaks target-specific `?=` into a + @# global override that clobbers the caller's environment to empty for + @# every OTHER target's recipe — breaking conformance-live's + @# required-env guards whenever this makefile is loaded. + @# + @# The rm -rf lives on its own recipe line, separate from the $(MAKE) + @# invocations: under `make -n`, lines containing $(MAKE) are still + @# executed (with -n propagated to the sub-make), so folding the rm into + @# that line would delete snapshots during a dry-run. + @# Guard against a catastrophic rm -rf: strip ALL trailing slashes (a + @# single strip would let '///' through as '//' ≈ root), then refuse + @# "", ".", "/" (repo checkout / filesystem root) and any '..' PATH + @# SEGMENT (leading, trailing, interior, or the whole path); finally, + @# canonicalize an EXISTING dir (cd + pwd -P) into a separate GUARD + @# variable for the checkout-ancestor check, so non-canonical + @# spellings ('/x//y', symlinks into the checkout) can't slip past the + @# string compare — a failed cd empties the guard value, which the + @# ancestor pattern then refuses. The rm itself uses the ORIGINAL + @# path: when LIVE_RECORD_DIR is a symlink, rm removes the link, not + @# the tree it points to. A '..' + @# inside a segment (tmp/live-canary..pr308) is benign and allowed. + @LRD="$${LIVE_RECORD_DIR:-tmp/live-canary}"; LRD_ORIG="$$LRD"; \ + while [ "$${LRD%/}" != "$$LRD" ]; do LRD="$${LRD%/}"; done; \ + case "$$LRD" in \ + ""|"."|"/") echo "ERROR: refusing rm -rf on unsafe LIVE_RECORD_DIR='$$LRD_ORIG'" >&2; exit 2 ;; \ + ".."|"../"*|*"/.."|*"/../"*) echo "ERROR: refusing rm -rf on LIVE_RECORD_DIR with a '..' path segment: '$$LRD_ORIG'" >&2; exit 2 ;; \ + esac; \ + LRD_CHECK="$$LRD"; \ + if [ -d "$$LRD" ]; then LRD_CHECK="$$(cd "$$LRD" && pwd -P)"; fi; \ + case "$$LRD_CHECK" in \ + "/") echo "ERROR: refusing rm -rf on LIVE_RECORD_DIR='$$LRD_ORIG' — it canonicalizes to the filesystem root" >&2; exit 2 ;; \ + esac; \ + case "$$(pwd -P)/" in \ + "$$LRD_CHECK"/*) echo "ERROR: refusing rm -rf on LIVE_RECORD_DIR='$$LRD_ORIG' — it is the repo checkout or one of its ancestors" >&2; exit 2 ;; \ + esac; \ + rm -rf -- "$$LRD" + @echo "==> check-bc5-compat: BC4 pass" + @LRD="$${LIVE_RECORD_DIR:-tmp/live-canary}"; \ + BC4_HOST="$${BASECAMP_HOST:-https://3.basecampapi.com}"; \ + BASECAMP_LIVE=1 BASECAMP_HOST="$$BC4_HOST" BASECAMP_BACKEND=bc4 LIVE_RECORD_DIR="$$LRD" $(MAKE) conformance-live + @echo "==> check-bc5-compat: BC5 pass" + @LRD="$${LIVE_RECORD_DIR:-tmp/live-canary}"; \ + BASECAMP_LIVE=1 BASECAMP_HOST="$$BC5_HOST" BASECAMP_BACKEND=bc5 LIVE_RECORD_DIR="$$LRD" $(MAKE) conformance-live + @echo "==> check-bc5-compat: pairwise BC4↔BC5 comparison" + @LRD="$${LIVE_RECORD_DIR:-tmp/live-canary}"; \ + ./scripts/compare-canary-runs.sh "$$LRD/bc4/wire" "$$LRD/bc5/wire" + #------------------------------------------------------------------------------ # Kotlin SDK targets #------------------------------------------------------------------------------ @@ -641,7 +747,7 @@ tools: # Spec-shape lints #------------------------------------------------------------------------------ -.PHONY: check-bucket-flat-parity validate-api-gaps +.PHONY: check-bucket-flat-parity validate-api-gaps check-compare-canary # Verify every bucket-scoped GET list operation has a flat-path counterpart # (or is justified in spec/bucket-scoped-allowlist.txt). Cross-project SDK @@ -653,6 +759,15 @@ check-bucket-flat-parity: validate-api-gaps: @./scripts/validate-api-gaps.sh +# Network-free regression tests for the pairwise compare script (filename +# scheme, missing-snapshot hard-fail, memories waiver scoping, pairwiseEqual +# key-order, empty-paths guard). +# +# Requires bash >= 4.4 on PATH (`env bash`): both scripts guard for it and +# fail with a clear message; macOS ships 3.2 at /bin/bash — brew install bash. +check-compare-canary: + @./scripts/test-compare-canary-runs.sh + #------------------------------------------------------------------------------ # Combined targets #------------------------------------------------------------------------------ @@ -675,7 +790,7 @@ generate: @echo "==> Generation complete" # Run all checks (Smithy + Go + TypeScript + Ruby + Kotlin + Swift + Python + Behavior Model + Conformance + Provenance + Actions lint) -check: lint-actions sync-spec-version-check smithy-check behavior-model-check provenance-check sync-api-version-check go-check-drift go-check-wrapper-drift auth-routable-check kt-check-drift go-check ts-check rb-check kt-check swift-check py-check conformance check-bucket-flat-parity validate-api-gaps +check: lint-actions sync-spec-version-check smithy-check behavior-model-check provenance-check sync-api-version-check go-check-drift go-check-wrapper-drift auth-routable-check kt-check-drift go-check ts-check rb-check kt-check swift-check py-check conformance check-bucket-flat-parity validate-api-gaps check-compare-canary @echo "==> All checks passed" # Clean all build artifacts diff --git a/conformance/schema.json b/conformance/schema.json index 29d5e8a85..0fa182fbe 100644 --- a/conformance/schema.json +++ b/conformance/schema.json @@ -201,6 +201,45 @@ "description": "Placeholder names referenced in pathParams/queryParams for live tests (e.g., PROJECT_ID, TODOLIST_ID). Resolved per the fixture-ID ladder: explicit env var → generic env var → discovery → skip.", "additionalProperties": { "type": "string" } }, + "pairwiseAssertions": { + "type": "array", + "description": "Rules that compare a BC4 wire snapshot to a BC5 wire snapshot of the same operation. Paths are dotted, relative to each snapshot body — equivalent to `pages[0].body.` for single-page bodies; use explicit `pages[N].body.` to address a specific page or `pages[*].body.` to aggregate across pages. The empty path `\"\"` addresses the body root. These rules enforce the canary's additive-only invariant: BC5 must not regress fields/keys/values that BC4 emitted.", + "items": { + "type": "object", + "required": ["type", "paths"], + "properties": { + "type": { + "type": "string", + "enum": [ + "pairwiseSupersetArray", + "pairwiseSupersetKeys", + "pairwiseEqual", + "pairwiseDeltaAllowed" + ], + "description": "pairwiseSupersetArray: BC5 array length at each path must be >= BC4's. pairwiseSupersetKeys: BC5 object's keys at each path must be a superset of BC4's keys. pairwiseEqual: BC5 value at each path must equal BC4's (use sparingly; intended for type discriminators). pairwiseDeltaAllowed: paths where divergence is explicitly accepted; requires `reason`." + }, + "paths": { + "type": "array", + "minItems": 1, + "items": { "type": "string" }, + "description": "Dotted-path identifiers. Empty string \"\" addresses the body root. At least one path is required — an empty array is a meaningless rule and is rejected (the compare script would otherwise risk treating it as the body root)." + }, + "reason": { + "type": "string", + "description": "Free-form justification. Required for `pairwiseDeltaAllowed` (a public compatibility report indexes accepted divergences). Recommended for other rule types so the reader of a future failure knows the invariant being enforced." + } + }, + "allOf": [ + { + "if": { "properties": { "type": { "const": "pairwiseDeltaAllowed" } } }, + "then": { + "required": ["reason"], + "properties": { "reason": { "minLength": 1 } } + } + } + ] + } + }, "tags": { "type": "array", "description": "Tags for filtering tests (e.g., retry, pagination, idempotent)", diff --git a/conformance/tests/live-my-surface.json b/conformance/tests/live-my-surface.json index 1518f5362..779cc8db1 100644 --- a/conformance/tests/live-my-surface.json +++ b/conformance/tests/live-my-surface.json @@ -71,7 +71,7 @@ { "mode": "live", "name": "GetMyNotifications decodes unreads/reads/memories/bubble_ups", - "description": "Verifies BC5 additions (bubble_ups, scheduled_bubble_ups) decode as arrays. Also exercises the memories[] observation that pairs with the COORDINATION.md decision.", + "description": "Verifies BC5 additions (bubble_ups, scheduled_bubble_ups) decode as arrays, and exercises the top-level memories[] contract divergence: BC5 ships memories as a documented always-empty placeholder (doc/api/sections/my_notifications.md) while BC4 (the four branch) still populates it. The pairwiseSupersetArray rule below encodes the additive-only invariant; the pairwiseDeltaAllowed entry records the permanent, documented BC4→BC5 delta — see spec/api-gaps/memories-emptied-regression.md.", "operation": "GetMyNotifications", "method": "GET", "path": "/my/readings.json", @@ -79,6 +79,23 @@ "liveAssertions": [ { "type": "liveCallSucceeds" }, { "type": "liveSchemaValidate" } + ], + "pairwiseAssertions": [ + { + "type": "pairwiseSupersetArray", + "paths": ["memories"], + "reason": "Additive-only invariant: BC5 memories[] should remain a superset of BC4's. PERMANENTLY waived by the pairwiseDeltaAllowed entry below — bc3 documents memories as an always-empty placeholder on BC5 (doc/api/sections/my_notifications.md). See spec/api-gaps/memories-emptied-regression.md." + }, + { + "type": "pairwiseSupersetKeys", + "paths": [""], + "reason": "Top-level GetMyNotifications keys must not regress between BC4 and BC5 — additive-only invariant. New BC5 keys (bubble_ups, scheduled_bubble_ups) are expected; key removals fail. memories remains present as a key on BC5 (only its contents are emptied), so this rule is unaffected by the memories regression." + }, + { + "type": "pairwiseDeltaAllowed", + "paths": ["memories"], + "reason": "PERMANENT documented contract divergence: BC5 ships `json.memories []` by contract — doc/api/sections/my_notifications.md (bc3 #11628) codifies memories as an always-empty placeholder superseded by bubble_ups. The once-planned alias (`json.memories @bubble_ups`, bc3 #10947) never shipped; #10947 is closed unmerged. Not a pending regression — this entry is the machine-readable record of an accepted BC4→BC5 subtractive delta. Retire only if BC4 (the four branch) empties memories too (the superset rule then passes clean) or BC5's documented contract changes. Tracked in spec/api-gaps/memories-emptied-regression.md." + } ] }, { diff --git a/scripts/compare-canary-runs.sh b/scripts/compare-canary-runs.sh new file mode 100755 index 000000000..34fcfa79a --- /dev/null +++ b/scripts/compare-canary-runs.sh @@ -0,0 +1,544 @@ +#!/usr/bin/env bash +# Pairwise BC4↔BC5 wire-snapshot comparison. +# +# Reads two snapshot directories (one per backend) written by the TS live +# canary runner — `//wire/.json` — and applies +# the `pairwiseAssertions` rules from each test's entry in +# `conformance/tests/live-my-surface.json`. Reports violations of the +# additive-only invariant: BC5 must not drop arrays/keys/values that BC4 +# emitted, except where `pairwiseDeltaAllowed` explicitly accepts the drift. +# +# Path semantics +# -------------- +# Each rule's `paths` entries are dotted identifiers relative to a page body. +# +# - Empty string `""` addresses the body root. +# - `foo.bar` defaults to `pages[0].body.foo.bar` for single-page snapshots. +# - Paths starting with `pages[` are taken absolute, useful when a test +# captures multiple pages or wants to address a specific page index. +# +# Rule types +# ---------- +# - pairwiseSupersetArray: BC5 array length at each path must be ≥ BC4's. +# Catches "memories went to []". For aggregated +# `pages[*]` paths the comparison is the TOTAL item +# count across pages (a page missing the field +# contributes 0; a non-null non-array page value is +# invalid) — never the page count itself, which +# would false-green a field dropped on every page. +# - pairwiseSupersetKeys: BC5 object's keys at each path must be ⊇ BC4's. +# Catches "field disappeared from BC5". +# - pairwiseEqual: BC5 value at each path must equal BC4's. Use sparingly. +# - pairwiseDeltaAllowed: paths where BC5↔BC4 divergence is explicitly +# accepted; the listed paths are skipped by the +# other three rules for this operation. `reason` +# is required. +# +# Exit codes +# ---------- +# 0 clean: every rule held, or violations were covered by pairwiseDeltaAllowed. +# 1 one or more pairwise violations. +# 2 operator error: missing directory, missing test fixture, a missing or +# malformed wire snapshot for a pairwise test, jq unavailable, etc. +# +# Usage +# ----- +# compare-canary-runs.sh [tests-file] +# +# Path to the BC4 wire/ directory, e.g. +# tmp/live-canary/bc4/wire +# Path to the BC5 wire/ directory. +# [tests-file] Optional path to live-my-surface.json. Defaults to +# conformance/tests/live-my-surface.json relative to +# the script's project root. +# +# Comparison requires identical account state across the two runs. The +# CONTRIBUTING.md "Live canary" section documents this — without it, +# `unreads` and similar collections drift naturally and rules will false-fail. +set -euo pipefail + +# mapfile with NUL delimiters (bash 4.4+) is used below; macOS ships bash 3.2 +# at /bin/bash, where the failure mode is an opaque "mapfile: command not found". +if [ -z "${BASH_VERSINFO:-}" ] || [ "${BASH_VERSINFO[0]}" -lt 4 ] \ + || { [ "${BASH_VERSINFO[0]}" -eq 4 ] && [ "${BASH_VERSINFO[1]}" -lt 4 ]; }; then + echo "ERROR: bash >= 4.4 is required (found ${BASH_VERSION:-unknown}). On macOS: brew install bash" >&2 + exit 2 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then + echo "Usage: $0 [tests-file]" >&2 + exit 2 +fi + +BC4_DIR="$1" +BC5_DIR="$2" +TESTS_FILE="${3:-$PROJECT_ROOT/conformance/tests/live-my-surface.json}" + +if ! command -v jq >/dev/null 2>&1; then + echo "ERROR: jq is required" >&2 + exit 2 +fi + +for arg in "$BC4_DIR" "$BC5_DIR"; do + if [ ! -d "$arg" ]; then + echo "ERROR: snapshot directory not found: $arg" >&2 + exit 2 + fi +done + +if [ ! -f "$TESTS_FILE" ]; then + echo "ERROR: tests file not found: $TESTS_FILE" >&2 + exit 2 +fi + +# Normalize a user-supplied path to a jq path expression. +# +# Inputs → jq path +# "" → .pages[0].body +# "foo.bar" → .pages[0].body.foo.bar +# "pages[0].body.foo" → .pages[0].body.foo +# "pages[*].body.foo" → [.pages[].body.foo] (jq stream collapsed to array) +to_jq_path() { + local raw="$1" + if [ -z "$raw" ]; then + echo ".pages[0].body" + elif [[ "$raw" == pages\[* ]]; then + # Absolute path. Convert pages[*] to .pages[] (stream); we'll collect + # downstream by wrapping into [...] if a `*` is present. + local jq_path=".${raw//\[\*\]/[]}" + echo "$jq_path" + else + echo ".pages[0].body.$raw" + fi +} + +# Snapshot reads that fail (malformed JSON, unreadable file) are operator +# errors, not pairwise violations. Remap jq's exit status (often 4 or 5) to +# the documented exit code 2 so the 0/1/2 contract holds under `set -e`. +snapshot_read_error() { + echo "ERROR: failed to read snapshot '$1' at path '$2' (malformed snapshot JSON, or the rule's path doesn't form a valid jq expression)" >&2 + exit 2 +} + +# Read a JSON value at a normalized jq path from a snapshot file. +# Streams from pages[*] are wrapped into an array so the caller can treat +# them as a single aggregated value. +read_value() { + local snapshot="$1" + local user_path="$2" + local jq_path + jq_path="$(to_jq_path "$user_path")" + + if [[ "$user_path" == *"[*]"* ]]; then + jq -c "[ $jq_path ]" "$snapshot" || snapshot_read_error "$snapshot" "$user_path" + else + jq -c "$jq_path" "$snapshot" || snapshot_read_error "$snapshot" "$user_path" + fi +} + +# Structural validation of a wire snapshot: valid JSON alone isn't enough — +# `{}` or `{"pages": {}}` would make every read_value return null, silently +# false-greening superset rules (null counts as absent/empty). Enforce the +# TS live runner's contract: object with a non-empty pages array whose +# length matches pages_count. +validate_snapshot() { + local snap="$1" + if ! jq -e ' + type == "object" + and ((.pages | type) == "array") + and ((.pages | length) > 0) + and (.pages_count == (.pages | length)) + and all(.pages[]; + type == "object" + and (.status | type == "number") + and (.headers | type == "object") + and has("body") + and (.bodyText | type == "string") + and (.url | type == "string")) + ' "$snap" >/dev/null 2>&1; then + echo "ERROR: structurally invalid wire snapshot '$snap' (expected an object with a non-empty pages array, matching pages_count, and correctly typed {status: number, headers: object, body, bodyText: string, url: string} on every page)" >&2 + exit 2 + fi +} + +# Content-free shape summary of a JSON value, for violation messages. CI run +# logs are retained and readable by anyone with log access, so violations +# must never echo live-account field content — only type + size. +shape_of() { + jq -nr --argjson v "$1" '$v | if type == "object" then "object(\(keys | length) keys)" elif type == "array" then "array(\(length))" elif type == "string" then "string(\(length))" else "\(type)" end' +} + +# Bracket-path validation shared by enforcing rules and waiver entries. +# Brackets are only documented (and only handled) as a leading 'pages[N]' or +# 'pages[*]' segment, and a bracketed path must then enter the page BODY +# ('pages[N].body...'): page-level keys other than body (status, headers, +# url) are backend-specific metadata with no pairwise semantics, and a typo +# like 'pages[0].items' would read null on BOTH snapshots and false-green +# the rule. Any other bracket use — items[*].foo, a second star, a bare jq +# stream like items[] — would stream through jq with undefined comparison +# semantics. Reject all of these as fixture mistakes. +validate_rule_path() { + local upath="$1" ctx="$2" + # "" is the documented body-root sentinel. + [ -z "$upath" ] && return 0 + # WHITELIST, not blacklist: a path is dotted key segments ([A-Za-z0-9_]), + # optionally led by a pages[N]/pages[*] indexer that must then enter the + # page BODY. Anything else — commas, pipes, quotes, spaces, stray + # brackets, bare jq streams — would be interpolated into the jq program + # as raw syntax with undefined comparison semantics (page-level keys + # other than body are backend-specific metadata; a typo like + # 'pages[0].items' reads null on BOTH snapshots and false-greens rules). + # No '-' in segments: '.foo-bar' parses in jq as subtraction ('.foo - bar'), + # not a key lookup. No current fixture path needs it; keys requiring + # quoting are a future, deliberate extension of to_jq_path. + local re_pages='^pages\[([0-9]+|\*)\]\.body(\.[A-Za-z0-9_]+)*$' + local re_plain='^[A-Za-z0-9_]+(\.[A-Za-z0-9_]+)*$' + if [[ "$upath" =~ $re_pages || "$upath" =~ $re_plain ]]; then + return 0 + fi + echo "ERROR: unsupported path '$upath' on $ctx — paths are dotted [A-Za-z0-9_] key segments; brackets are only supported as the leading 'pages[N]' or 'pages[*]' segment and must enter the page body ('pages[N].body...')" >&2 + exit 2 +} + +# Path → display string for error messages. +display_path() { + local raw="$1" + if [ -z "$raw" ]; then + echo "" + else + echo "$raw" + fi +} + +VIOLATIONS="" +violation() { + VIOLATIONS="${VIOLATIONS}$1 +" +} + +# Collect comparable operations: tests that have pairwiseAssertions. +# Materialize the jq output before mapfile: a jq failure inside a process +# substitution isn't seen by set -e, so a malformed tests file would leave +# TEST_ENTRIES empty and the script would exit 0 ("nothing to compare"), +# silently masking an operator error. +if ! ENTRIES_RAW="$(jq -c 'if type != "array" then error("tests file must be a top-level array") else . end | .[] | select((.pairwiseAssertions // []) | length > 0)' "$TESTS_FILE")"; then + echo "ERROR: failed to parse tests file '$TESTS_FILE' (malformed JSON or not a top-level array)" >&2 + exit 2 +fi +mapfile -t TEST_ENTRIES <<<"$ENTRIES_RAW" +if [ "${#TEST_ENTRIES[@]}" -eq 1 ] && [ -z "${TEST_ENTRIES[0]}" ]; then + TEST_ENTRIES=() +fi + +if [ "${#TEST_ENTRIES[@]}" -eq 0 ]; then + echo "==> Pairwise canary: no tests carry pairwiseAssertions; nothing to compare" + exit 0 +fi + +COMPARED=0 +MISSING_SNAPSHOTS=0 + +for entry in "${TEST_ENTRIES[@]}"; do + NAME="$(jq -r '.name' <<<"$entry")" + OPERATION="$(jq -r '.operation' <<<"$entry")" + + # Snapshot filenames must match exactly what the TS live runner writes. + # conformance/runner/typescript/live-runner.test.ts persistSnapshot() uses + # safeName = testName.replace(/[^a-z0-9_-]+/gi, "_") + # i.e. the TEST NAME (not the operation), case preserved, with each run of + # characters outside [A-Za-z0-9_-] collapsed to a single "_". There is no + # operation-based or lowercased filename — the earlier candidate forms never + # matched a real file, so every comparison was silently skipped. + safe="$(printf '%s' "$NAME" | sed -E 's/[^a-zA-Z0-9_-]+/_/g')" + BC4_SNAPSHOT="$BC4_DIR/$safe.json" + BC5_SNAPSHOT="$BC5_DIR/$safe.json" + + # TEST_ENTRIES contains only tests that declare pairwiseAssertions, so a + # missing snapshot on either backend is a hard error (exit 2): skipping it + # would let check-bc5-compat pass without ever evaluating the declared rule. + if [ ! -f "$BC4_SNAPSHOT" ] || [ ! -f "$BC5_SNAPSHOT" ]; then + MISSING_SNAPSHOTS=$((MISSING_SNAPSHOTS + 1)) + [ ! -f "$BC4_SNAPSHOT" ] && echo "ERROR: missing BC4 snapshot for test '$NAME' ($OPERATION): $BC4_SNAPSHOT" >&2 + [ ! -f "$BC5_SNAPSHOT" ] && echo "ERROR: missing BC5 snapshot for test '$NAME' ($OPERATION): $BC5_SNAPSHOT" >&2 + continue + fi + + validate_snapshot "$BC4_SNAPSHOT" + validate_snapshot "$BC5_SNAPSHOT" + + # Snapshot ↔ test integrity: a stale or overwritten snapshot (matching + # filename, different operation) must not be compared as if it were this + # test's capture. + for snap in "$BC4_SNAPSHOT" "$BC5_SNAPSHOT"; do + SNAP_OP="$(jq -r '.operation // ""' "$snap")" + if [ "$SNAP_OP" != "$OPERATION" ]; then + echo "ERROR: snapshot '$snap' records operation '$SNAP_OP' but test '$NAME' expects '$OPERATION' (stale or overwritten snapshot?)" >&2 + exit 2 + fi + done + + COMPARED=$((COMPARED + 1)) + + # Allowlisted paths for this operation (skipped by the other rule types). + # Materialize jq output before mapfile (a process substitution's failure is + # invisible to set -e), and drive emptiness off jq's own length so a lone + # empty-string path ("" = body root) isn't confused with no paths at all. + if ! ALLOW_JSON="$(jq -c ' + (.pairwiseAssertions // []) + | map(select(.type == "pairwiseDeltaAllowed")) + | if all(.[]; (.reason | type == "string" and length > 0)) then . + else error("pairwiseDeltaAllowed requires a non-empty reason") + end + | map(.paths // []) + | if all(.[]; type == "array" and length > 0 and all(.[]; type == "string")) + then flatten + else error("pairwiseDeltaAllowed paths must be non-empty arrays of strings") + end + ' <<<"$entry")"; then + echo "ERROR: invalid pairwiseDeltaAllowed waiver on $OPERATION — each waiver needs a non-empty 'reason' (accepted divergences must be audited) and 'paths' as a non-empty array of strings (schema minItems: 1; a bare-string paths would silently suppress enforcement)" >&2 + exit 2 + fi + ALLOW_COUNT="$(jq -r 'length' <<<"$ALLOW_JSON")" + ALLOW_PATHS=() + if [ "$ALLOW_COUNT" -gt 0 ]; then + # NUL-delimited extraction: newline splitting via command substitution + # strips trailing blank lines, silently dropping a trailing empty-string + # path ("" = body root). The count check backstops jq failures inside + # the process substitution (invisible to set -e) and non-string elements. + mapfile -d '' -t ALLOW_PATHS < <(jq -j '.[] + "\u0000"' <<<"$ALLOW_JSON") + if [ "${#ALLOW_PATHS[@]}" -ne "$ALLOW_COUNT" ]; then + echo "ERROR: pairwiseDeltaAllowed path extraction mismatch for $OPERATION (expected $ALLOW_COUNT, got ${#ALLOW_PATHS[@]}; paths must be strings)" >&2 + exit 2 + fi + # A malformed bracket path in a waiver-only entry must be a fixture + # error too — it names a path no enforcing rule can ever target, so it + # would otherwise sit unnoticed while appearing to document a waiver. + for ap in "${ALLOW_PATHS[@]}"; do + validate_rule_path "$ap" "$OPERATION (pairwiseDeltaAllowed)" + done + fi + is_allowed() { + local p="$1" + local ap + # Use ${#var[@]} guard rather than ${var[@]:-} — the latter substitutes a + # single empty string for an empty array, which would erroneously match + # an empty-string `""` rule path (the "body root" sentinel). + if [ "${#ALLOW_PATHS[@]}" -eq 0 ]; then + return 1 + fi + for ap in "${ALLOW_PATHS[@]}"; do + [ "$p" = "$ap" ] && return 0 + done + return 1 + } + + # Iterate over the enforcing rules (everything except pairwiseDeltaAllowed). + # Same materialize-before-mapfile treatment as ALLOW_PATHS above. + if ! RULES_RAW="$(jq -c ' + (.pairwiseAssertions // []) + | map(select(.type != "pairwiseDeltaAllowed")) + | .[] + ' <<<"$entry")"; then + echo "ERROR: failed to extract pairwise rules for $OPERATION" >&2 + exit 2 + fi + mapfile -t ENFORCED_RULES <<<"$RULES_RAW" + if [ "${#ENFORCED_RULES[@]}" -eq 1 ] && [ -z "${ENFORCED_RULES[0]}" ]; then + ENFORCED_RULES=() + fi + + for rule in "${ENFORCED_RULES[@]}"; do + RULE_TYPE="$(jq -r '.type' <<<"$rule")" + + # Reject unknown rule types up front, BEFORE any per-path processing: + # the dispatch case below only runs for paths that survive the waiver + # skip, so a misspelled type whose paths are all waived would otherwise + # exit clean — an operator error silently suppressed. This script is + # invoked directly without schema validation, so it must self-check. + case "$RULE_TYPE" in + pairwiseSupersetArray | pairwiseSupersetKeys | pairwiseEqual) ;; + *) + echo "ERROR: unknown pairwise rule type '$RULE_TYPE' on $OPERATION — schema validation should have caught this" >&2 + exit 2 + ;; + esac + + # Guard the empty-array case off jq's own length, BEFORE splitting into + # lines: an empty `paths` must not run the rule against the body root + # (schema.json enforces minItems:1; this is defense in depth), and a lone + # empty-string path ("" = body root) must not be mistaken for empty. + RP_COUNT="$(jq -r '(.paths // []) | if type == "array" then length else "INVALID" end' <<<"$rule")" + if [ "$RP_COUNT" = "INVALID" ]; then + echo "ERROR: 'paths' for $RULE_TYPE rule on $OPERATION must be an array of strings" >&2 + exit 2 + fi + if [ "$RP_COUNT" -eq 0 ]; then + echo "ERROR: $RULE_TYPE rule on $OPERATION has an empty 'paths' array" >&2 + exit 2 + fi + # NUL-delimited extraction preserves a trailing empty-string path ("" = + # body root), which newline splitting would strip. The count check + # backstops jq failures inside the process substitution (invisible to + # set -e) and non-string elements. + mapfile -d '' -t RULE_PATHS < <(jq -j '.paths[] + "\u0000"' <<<"$rule") + if [ "${#RULE_PATHS[@]}" -ne "$RP_COUNT" ]; then + echo "ERROR: 'paths' extraction mismatch for $RULE_TYPE rule on $OPERATION (expected $RP_COUNT, got ${#RULE_PATHS[@]}; paths must be strings)" >&2 + exit 2 + fi + + for upath in "${RULE_PATHS[@]}"; do + # Validate BEFORE the waiver skip below: a waived-but-unsupported + # path must still be reported, or a typo in a waived canary rule + # would permanently suppress enforcement without the fixture error + # ever surfacing. + validate_rule_path "$upath" "$OPERATION" + + # A concrete pages[N] rule asserts "page N exists on both backends" + # as a precondition. If either capture has fewer pages, reading the + # path yields null and the rule would pass clean — hiding a fixture + # typo (pages[1] against single-page captures) or a page-count + # regression. Both demand operator attention; fail as a fixture/ + # capture error rather than comparing nulls. Cross-page-count + # comparison is what 'pages[*]' aggregation is for. + if [[ "$upath" =~ ^pages\[([0-9]+)\] ]]; then + PAGE_IDX="${BASH_REMATCH[1]}" + for side_snap in "$BC4_SNAPSHOT:bc4" "$BC5_SNAPSHOT:bc5"; do + snap_file="${side_snap%:*}"; snap_label="${side_snap##*:}" + snap_pages="$(jq -r '.pages | length' "$snap_file")" + if [ "$PAGE_IDX" -ge "$snap_pages" ]; then + echo "ERROR: path '$upath' on $OPERATION targets page $PAGE_IDX but the $snap_label snapshot has only $snap_pages page(s) — fix the rule's page index, or use 'pages[*]' to aggregate across pages" >&2 + exit 2 + fi + done + elif [[ "$upath" != pages\[* ]]; then + # Shorthand paths ("" or "foo.bar") normalize to pages[0].body — a + # deliberate convenience for single-page captures only. Against a + # multi-page snapshot the shorthand would silently check page 0 and + # ignore the rest, under-enforcing the rule. Require explicit + # pages[N]/pages[*] syntax as soon as either capture paginates. + for side_snap in "$BC4_SNAPSHOT:bc4" "$BC5_SNAPSHOT:bc5"; do + snap_file="${side_snap%:*}"; snap_label="${side_snap##*:}" + snap_pages="$(jq -r '.pages | length' "$snap_file")" + if [ "$snap_pages" -gt 1 ]; then + echo "ERROR: shorthand path '$(display_path "$upath")' on $OPERATION is ambiguous — the $snap_label snapshot has $snap_pages pages; use explicit 'pages[N].body...' or 'pages[*].body...' syntax" >&2 + exit 2 + fi + done + fi + + if is_allowed "$upath"; then + continue + fi + + DISPLAY="$(display_path "$upath")" + BC4_VAL="$(read_value "$BC4_SNAPSHOT" "$upath")" + BC5_VAL="$(read_value "$BC5_SNAPSHOT" "$upath")" + + case "$RULE_TYPE" in + pairwiseSupersetArray) + # null at a path means "field absent on this backend". A BC4 array + # of N items vs BC5 null is a regression; treat null as length 0 + # only when there's nothing on either side. + # + # Aggregated `pages[*]` paths wrap one value per page, so comparing + # the outer length would just compare page counts — a field dropped + # on every page would false-green. Compare the total item count + # across pages instead: an absent (null) page value contributes 0, + # any non-null non-array page value poisons the total as INVALID. + if [[ "$upath" == *"[*]"* ]]; then + AGG_LEN='[ .[] | if . == null then 0 elif type == "array" then length else "INVALID" end ] + | if any(.[]; . == "INVALID") then "INVALID" else (add // 0) end' + BC4_LEN="$(jq -r "$AGG_LEN" <<<"$BC4_VAL")" + BC5_LEN="$(jq -r "$AGG_LEN" <<<"$BC5_VAL")" + else + BC4_LEN="$(jq -r 'if type == "array" then length else (if . == null then 0 else "INVALID" end) end' <<<"$BC4_VAL")" + BC5_LEN="$(jq -r 'if type == "array" then length else (if . == null then 0 else "INVALID" end) end' <<<"$BC5_VAL")" + fi + + if [ "$BC4_LEN" = "INVALID" ] || [ "$BC5_LEN" = "INVALID" ]; then + violation "$OPERATION pairwiseSupersetArray($DISPLAY): expected arrays on both sides; BC4 $(shape_of "$BC4_VAL") vs BC5 $(shape_of "$BC5_VAL") (raw values redacted from logs)" + elif [ "$BC5_LEN" -lt "$BC4_LEN" ]; then + violation "$OPERATION pairwiseSupersetArray($DISPLAY): BC5 length $BC5_LEN < BC4 length $BC4_LEN" + fi + ;; + + pairwiseSupersetKeys) + # Missing (null) on either side counts as the empty key set. Any + # other non-object is an invalid target for a keys rule — flag it + # instead of silently skipping, so a mis-specified path or a real + # shape change (object → array/scalar) can't hide. + BC4_KIND="$(jq -r 'if type == "object" then "object" elif . == null then "null" else "INVALID" end' <<<"$BC4_VAL")" + BC5_KIND="$(jq -r 'if type == "object" then "object" elif . == null then "null" else "INVALID" end' <<<"$BC5_VAL")" + + if [ "$BC4_KIND" = "INVALID" ] || [ "$BC5_KIND" = "INVALID" ]; then + violation "$OPERATION pairwiseSupersetKeys($DISPLAY): expected objects (or null for absent) on both sides; BC4 $(shape_of "$BC4_VAL") vs BC5 $(shape_of "$BC5_VAL") (raw values redacted from logs)" + else + BC4_OBJ="$BC4_VAL" + BC5_OBJ="$BC5_VAL" + if [ "$BC4_KIND" = "null" ]; then BC4_OBJ="{}"; fi + if [ "$BC5_KIND" = "null" ]; then BC5_OBJ="{}"; fi + MISSING="$(jq -r --argjson bc5 "$BC5_OBJ" ' + keys + | map(select(. as $k | ($bc5 | has($k)) | not)) + | join(",") + ' <<<"$BC4_OBJ")" + if [ -n "$MISSING" ]; then + violation "$OPERATION pairwiseSupersetKeys($DISPLAY): BC5 missing keys present in BC4: $MISSING" + fi + fi + ;; + + pairwiseEqual) + # Compare semantically: jq deep-equality is independent of object + # key order, so two snapshots that serialize the same object with + # different key order don't false-fail. REDACT the values in the + # violation: these are live-account response fields, and CI run + # logs are retained and readable by anyone with log access — + # report only shape summaries (type, and keys/length/digest), not + # content. Operators reproduce locally against their own capture + # to see the raw values. + if [ "$(jq -n --argjson a "$BC4_VAL" --argjson b "$BC5_VAL" '$a == $b')" != "true" ]; then + violation "$OPERATION pairwiseEqual($DISPLAY): values differ — BC4 $(shape_of "$BC4_VAL") vs BC5 $(shape_of "$BC5_VAL") (raw values redacted from logs; re-run the comparison locally to inspect)" + fi + ;; + + *) + echo "ERROR: unknown pairwise rule type '$RULE_TYPE' on $OPERATION — schema validation should have caught this" >&2 + exit 2 + ;; + esac + done + done +done + +echo "==> Pairwise canary: compared $COMPARED operation(s)" + +if [ "$MISSING_SNAPSHOTS" -gt 0 ]; then + echo "" >&2 + echo "ERROR: $MISSING_SNAPSHOTS pairwise test(s) were missing a wire snapshot on" >&2 + echo "one or both backends (listed above). The TS live runner must capture every" >&2 + echo "pairwise-bearing test for both BC4 and BC5 before comparison; a missing" >&2 + echo "snapshot is a hard error so check-bc5-compat can't report a false green" >&2 + echo "without evaluating the declared rule." >&2 + if [ -n "$VIOLATIONS" ]; then + echo "" >&2 + echo "Pairwise violations were also found in the snapshots that were present:" >&2 + printf '%s' "$VIOLATIONS" >&2 + fi + exit 2 +fi + +if [ -n "$VIOLATIONS" ]; then + echo "" >&2 + echo "Additive-only invariant violated:" >&2 + printf '%s' "$VIOLATIONS" >&2 + echo "" >&2 + echo "If a divergence is intentional, add a pairwiseDeltaAllowed entry on" >&2 + echo "the operation in live-my-surface.json with a reason." >&2 + exit 1 +fi + +echo "Pairwise canary clean" diff --git a/scripts/test-compare-canary-runs.sh b/scripts/test-compare-canary-runs.sh new file mode 100755 index 000000000..93034ed76 --- /dev/null +++ b/scripts/test-compare-canary-runs.sh @@ -0,0 +1,992 @@ +#!/usr/bin/env bash +# Regression tests for scripts/compare-canary-runs.sh. +# +# Network-free: builds synthetic wire snapshots in a temp dir and asserts the +# compare script's exit code + key output for each scenario. Guards the bug +# classes fixed for PR #308: +# +# - P0 snapshot filename must match the TS live runner's scheme exactly +# (testName.replace(/[^a-z0-9_-]+/gi, "_"), case-preserved). The old +# lowercase/hyphen/operation-first form found nothing and silently +# passed. +# - P1 a declared pairwise test with a missing snapshot is a hard error, +# not a silent skip (which would let check-bc5-compat false-green). +# - the `memories` pairwiseDeltaAllowed waiver scopes to `memories` only; +# an unrelated regression still fails. +# - pairwiseEqual compares semantically (object key order is irrelevant). +# - an empty `paths` array is rejected at runtime (defense in depth behind +# the schema's minItems:1). +# - pairwiseSupersetKeys flags non-object values instead of silently +# skipping; null (absent) counts as the empty key set on either side. +set -euo pipefail + +# Fail fast with one clear message rather than sixteen confusing scenario +# failures: compare-canary-runs.sh requires bash >= 4.4 (mapfile -d), and on +# macOS `/usr/bin/env bash` may resolve to the system 3.2. +if [ -z "${BASH_VERSINFO:-}" ] || [ "${BASH_VERSINFO[0]}" -lt 4 ] \ + || { [ "${BASH_VERSINFO[0]}" -eq 4 ] && [ "${BASH_VERSINFO[1]}" -lt 4 ]; }; then + echo "ERROR: bash >= 4.4 is required (found ${BASH_VERSION:-unknown}). On macOS: brew install bash" >&2 + exit 2 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "ERROR: jq is required" >&2 + exit 2 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPARE="$SCRIPT_DIR/compare-canary-runs.sh" +REAL_TESTS="$SCRIPT_DIR/../conformance/tests/live-my-surface.json" + +# The exact filename the TS runner writes for the real GetMyNotifications test. +# Hardcoded (not re-derived from the sed in the compare script) so a regression +# in the sanitizer is actually caught: this is the golden value from +# conformance/runner/typescript/live-runner.test.ts persistSnapshot(). +GMN_TEST_NAME="GetMyNotifications decodes unreads/reads/memories/bubble_ups" +GMN_SAFE_NAME="GetMyNotifications_decodes_unreads_reads_memories_bubble_ups" + +# Template form for portability: current macOS mktemp accepts a bare -d, +# but older BSD variants insist on a template — and the template costs nothing. +TMP="$(mktemp -d "${TMPDIR:-/tmp}/compare-canary-tests.XXXXXX")" +trap 'rm -rf -- "$TMP"' EXIT + +# Scenarios A–D exercise the real GetMyNotifications entry (golden filename, +# memories waiver, missing-snapshot hard error). Filter that one entry out of +# the real fixture so adding more pairwise-bearing tests to live-my-surface.json +# doesn't break these scenarios with missing-snapshot errors for tests this +# suite never writes snapshots for. The entry itself stays verbatim-real. +GMN_TESTS="$TMP/gmn-tests.json" +jq --arg name "$GMN_TEST_NAME" 'map(select(.name == $name))' "$REAL_TESTS" >"$GMN_TESTS" +if [ "$(jq 'length' "$GMN_TESTS")" -ne 1 ]; then + echo "FATAL: expected exactly one '$GMN_TEST_NAME' entry in $REAL_TESTS" >&2 + exit 1 +fi + +FAILURES=0 +pass() { printf ' ok %s\n' "$1"; } +fail() { printf ' FAIL %s\n' "$1" >&2; FAILURES=$((FAILURES + 1)); } + +# write_snapshot +write_snapshot() { + local file="$1" operation="$2" body="$3" + mkdir -p "$(dirname "$file")" + jq -n --arg op "$operation" --argjson body "$body" \ + '{operation: $op, pages: [{status: 200, headers: {}, body: $body, bodyText: ($body | tostring), url: "https://example.test"}], pages_count: 1}' \ + >"$file" +} + +# write_snapshot_2p +write_snapshot_2p() { + local file="$1" operation="$2" body1="$3" body2="$4" + mkdir -p "$(dirname "$file")" + jq -n --arg op "$operation" --argjson b1 "$body1" --argjson b2 "$body2" \ + '{operation: $op, pages: [ + {status: 200, headers: {}, body: $b1, bodyText: ($b1 | tostring), url: "https://example.test?page=1"}, + {status: 200, headers: {}, body: $b2, bodyText: ($b2 | tostring), url: "https://example.test?page=2"} + ], pages_count: 2}' \ + >"$file" +} + +# fresh_dirs -> echoes " " (created empty) +fresh_dirs() { + local s="$1" + local bc4="$TMP/$s/bc4/wire" bc5="$TMP/$s/bc5/wire" + mkdir -p "$bc4" "$bc5" + echo "$bc4 $bc5" +} + +# run the compare script, capturing exit code + combined output +# usage: run_compare [tests-file] +RUN_OUT="" +RUN_RC=0 +run_compare() { + set +e + RUN_OUT="$("$COMPARE" "$@" 2>&1)" + RUN_RC=$? + set -e +} + +echo "==> test-compare-canary-runs" + +# --------------------------------------------------------------------------- +# Test A: TS filename scheme is found + evaluated, and the memories waiver +# allows BC5 memories[] to be shorter than BC4's. Uses the REAL fixture. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs A)" +write_snapshot "$BC4/$GMN_SAFE_NAME.json" GetMyNotifications \ + '{"unreads":[1,2],"reads":[3],"memories":[10,11,12],"bubble_ups":[]}' +write_snapshot "$BC5/$GMN_SAFE_NAME.json" GetMyNotifications \ + '{"unreads":[1,2],"reads":[3],"memories":[],"bubble_ups":[],"scheduled_bubble_ups":[]}' +run_compare "$BC4" "$BC5" "$GMN_TESTS" +if [ "$RUN_RC" -eq 0 ] && grep -q "compared 1 operation" <<<"$RUN_OUT"; then + pass "A: correct-named snapshot found + memories waiver applied (exit 0)" +else + fail "A: expected exit 0 with 'compared 1 operation'; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test B: the OLD filename form (lowercase + hyphen + operation-first) is NOT +# found, proving the scheme matters (this is the P0 regression made concrete). +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs B)" +OLD_NAME="getmynotifications-decodes-unreads-reads-memories-bubble_ups" +write_snapshot "$BC4/$OLD_NAME.json" GetMyNotifications '{"memories":[1]}' +write_snapshot "$BC5/$OLD_NAME.json" GetMyNotifications '{"memories":[]}' +run_compare "$BC4" "$BC5" "$GMN_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "missing .* snapshot" <<<"$RUN_OUT"; then + pass "B: old lowercase/hyphen filename is not found (exit 2)" +else + fail "B: expected exit 2 (missing) for old-scheme name; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test C (P1): a declared pairwise test missing a snapshot on one backend is a +# hard error, not a silent skip. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs C)" +write_snapshot "$BC4/$GMN_SAFE_NAME.json" GetMyNotifications '{"memories":[1,2,3]}' +# (no BC5 snapshot written) +run_compare "$BC4" "$BC5" "$GMN_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "missing BC5 snapshot" <<<"$RUN_OUT"; then + pass "C: missing BC5 snapshot hard-fails (exit 2), no silent skip" +else + fail "C: expected exit 2 with 'missing BC5 snapshot'; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test D: the memories waiver scopes to `memories` only. A real regression on a +# different path (a dropped top-level key) still fails via pairwiseSupersetKeys. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs D)" +write_snapshot "$BC4/$GMN_SAFE_NAME.json" GetMyNotifications \ + '{"unreads":[1],"reads":[2],"memories":[10],"bubble_ups":[]}' +# BC5 drops the top-level "unreads" key entirely (a genuine regression). +write_snapshot "$BC5/$GMN_SAFE_NAME.json" GetMyNotifications \ + '{"reads":[2],"memories":[],"bubble_ups":[]}' +run_compare "$BC4" "$BC5" "$GMN_TESTS" +if [ "$RUN_RC" -eq 1 ] && grep -q "missing keys present in BC4: unreads" <<<"$RUN_OUT"; then + pass "D: dropped top-level key still fails (exit 1); waiver scoped to memories" +else + fail "D: expected exit 1 with dropped-key violation; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test E: pairwiseEqual compares semantically — object key order is irrelevant. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs E)" +EQ_TESTS="$TMP/E/eq-tests.json" +cat >"$EQ_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Eq order test", + "operation": "EqOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseEqual", "paths": ["obj"], "reason": "discriminator shape must match" } + ] + } +] +JSON +write_snapshot "$BC4/Eq_order_test.json" EqOp '{"obj":{"a":1,"b":2,"c":3}}' +write_snapshot "$BC5/Eq_order_test.json" EqOp '{"obj":{"c":3,"b":2,"a":1}}' +run_compare "$BC4" "$BC5" "$EQ_TESTS" +if [ "$RUN_RC" -eq 0 ]; then + pass "E: pairwiseEqual ignores object key order (exit 0)" +else + fail "E: expected exit 0 for key-order-only difference; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test F: an empty `paths` array is rejected at runtime (not run against the +# body root). Both snapshots present so missing-snapshot can't mask this. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs F)" +EMPTY_TESTS="$TMP/F/empty-tests.json" +cat >"$EMPTY_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Empty paths test", + "operation": "EpOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetKeys", "paths": [] } + ] + } +] +JSON +write_snapshot "$BC4/Empty_paths_test.json" EpOp '{"k":1}' +write_snapshot "$BC5/Empty_paths_test.json" EpOp '{"k":1}' +run_compare "$BC4" "$BC5" "$EMPTY_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "empty 'paths' array" <<<"$RUN_OUT"; then + pass "F: empty 'paths' array rejected at runtime (exit 2)" +else + fail "F: expected exit 2 with empty-paths error; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test G: pairwiseSupersetKeys flags a non-object value instead of silently +# skipping — a mis-specified path or an object → scalar shape change must not +# hide behind a no-op rule. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs G)" +SK_TESTS="$TMP/G/sk-tests.json" +cat >"$SK_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Keys type test", + "operation": "SkOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetKeys", "paths": ["obj"], "reason": "keys rule must reject non-object shapes" } + ] + } +] +JSON +write_snapshot "$BC4/Keys_type_test.json" SkOp '{"obj":{"a":1}}' +write_snapshot "$BC5/Keys_type_test.json" SkOp '{"obj":"not-an-object"}' +run_compare "$BC4" "$BC5" "$SK_TESTS" +if [ "$RUN_RC" -eq 1 ] && grep -q "expected objects" <<<"$RUN_OUT"; then + pass "G: non-object at a supersetKeys path fails (exit 1), no silent skip" +else + fail "G: expected exit 1 with type violation; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test H: null (absent) counts as the empty key set — BC5 null where BC4 has +# keys reports the missing keys rather than passing or erroring on type. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs H)" +NK_TESTS="$TMP/H/nk-tests.json" +cat >"$NK_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Keys null test", + "operation": "NkOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetKeys", "paths": ["obj"], "reason": "absent object on BC5 is a dropped-keys regression" } + ] + } +] +JSON +write_snapshot "$BC4/Keys_null_test.json" NkOp '{"obj":{"a":1,"b":2}}' +write_snapshot "$BC5/Keys_null_test.json" NkOp '{}' +run_compare "$BC4" "$BC5" "$NK_TESTS" +if [ "$RUN_RC" -eq 1 ] && grep -q "missing keys present in BC4: a,b" <<<"$RUN_OUT"; then + pass "H: BC5 null where BC4 has keys reports the missing keys (exit 1)" +else + fail "H: expected exit 1 with missing-keys a,b; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test I: a malformed tests file is an operator error (exit 2), not a silent +# "nothing to compare" pass — jq's failure must not vanish into the mapfile +# process substitution. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs I)" +BAD_TESTS="$TMP/I/bad-tests.json" +printf '{ this is not json' >"$BAD_TESTS" +run_compare "$BC4" "$BC5" "$BAD_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "failed to parse tests file" <<<"$RUN_OUT"; then + pass "I: malformed tests file fails with exit 2, not a silent pass" +else + fail "I: expected exit 2 with parse error; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test J: a tests file whose top level is an object (not an array) is also an +# operator error — jq's `.[]` would happily iterate an object's values, so the +# type is checked explicitly. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs J)" +OBJ_TESTS="$TMP/J/obj-tests.json" +printf '{"tests": []}' >"$OBJ_TESTS" +run_compare "$BC4" "$BC5" "$OBJ_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "failed to parse tests file" <<<"$RUN_OUT"; then + pass "J: top-level-object tests file fails with exit 2, not a silent pass" +else + fail "J: expected exit 2 for non-array top level; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test K: pages[*] superset arrays compare TOTAL items across pages, not the +# page count — BC4 with items on every page vs BC5 with the field dropped on +# every page must fail even though pages_count matches. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs K)" +AGG_TESTS="$TMP/K/agg-tests.json" +cat >"$AGG_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Agg pages test", + "operation": "AggOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["pages[*].body.items"], "reason": "items must not shrink across the full paginated collection" } + ] + } +] +JSON +write_snapshot_2p "$BC4/Agg_pages_test.json" AggOp '{"items":[1,2]}' '{"items":[3]}' +write_snapshot_2p "$BC5/Agg_pages_test.json" AggOp '{"other":true}' '{"other":true}' +run_compare "$BC4" "$BC5" "$AGG_TESTS" +if [ "$RUN_RC" -eq 1 ] && grep -q "BC5 length 0 < BC4 length 3" <<<"$RUN_OUT"; then + pass "K: pages[*] field dropped on every page fails on totals (exit 1)" +else + fail "K: expected exit 1 comparing totals 0 < 3; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test L: pages[*] totals tolerate redistribution — the same items split +# differently across pages is not a regression. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs L)" +write_snapshot_2p "$BC4/Agg_pages_test.json" AggOp '{"items":[1,2]}' '{"items":[3]}' +write_snapshot_2p "$BC5/Agg_pages_test.json" AggOp '{"items":[1]}' '{"items":[2,3]}' +run_compare "$BC4" "$BC5" "$AGG_TESTS" +if [ "$RUN_RC" -eq 0 ]; then + pass "L: pages[*] redistribution with total preserved passes (exit 0)" +else + fail "L: expected exit 0 for preserved total; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test M: a non-array `paths` (string/object) is an operator error (exit 2) +# with an explicit message — not jq's raw exit status and not a silent +# misinterpretation (an object's values would otherwise iterate as paths). +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs M)" +STR_TESTS="$TMP/M/str-tests.json" +cat >"$STR_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "String paths test", + "operation": "SpOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseEqual", "paths": "obj" } + ] + } +] +JSON +write_snapshot "$BC4/String_paths_test.json" SpOp '{"obj":1}' +write_snapshot "$BC5/String_paths_test.json" SpOp '{"obj":1}' +run_compare "$BC4" "$BC5" "$STR_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "must be an array of strings" <<<"$RUN_OUT"; then + pass "M: non-array 'paths' fails with exit 2 and explicit message" +else + fail "M: expected exit 2 with type error; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test N: a structurally invalid snapshot (valid JSON, wrong shape) is an +# operator error — '{}' would otherwise make every read return null and +# false-green superset rules. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs N)" +write_snapshot "$BC4/Eq_order_test.json" EqOp '{"obj":{"a":1}}' +printf '{}' >"$BC5/Eq_order_test.json" +run_compare "$BC4" "$BC5" "$EQ_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "structurally invalid wire snapshot" <<<"$RUN_OUT"; then + pass "N: structurally invalid snapshot fails with exit 2, not a false-green" +else + fail "N: expected exit 2 with structural error; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test O: a trailing empty-string path ("" = body root) in a paths list is +# preserved and evaluated — newline splitting would have dropped it and +# exit-2'd on a count mismatch for a perfectly valid rule. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs O)" +BR_TESTS="$TMP/O/br-tests.json" +cat >"$BR_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Body root trailing test", + "operation": "BrOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseEqual", "paths": ["obj", ""], "reason": "body root listed last must still be compared" } + ] + } +] +JSON +write_snapshot "$BC4/Body_root_trailing_test.json" BrOp '{"obj":1,"k":2}' +write_snapshot "$BC5/Body_root_trailing_test.json" BrOp '{"obj":1,"k":2}' +run_compare "$BC4" "$BC5" "$BR_TESTS" +if [ "$RUN_RC" -eq 0 ] && grep -q "compared 1 operation" <<<"$RUN_OUT"; then + pass "O: trailing body-root path is preserved and evaluated (exit 0)" +else + fail "O: expected exit 0 with comparison run; got rc=$RUN_RC: $RUN_OUT" +fi + +# O2: same rule must still FAIL when the body root actually differs, +# proving the trailing "" path is evaluated rather than merely tolerated. +read -r BC4 BC5 <<<"$(fresh_dirs O2)" +write_snapshot "$BC4/Body_root_trailing_test.json" BrOp '{"obj":1,"k":2}' +write_snapshot "$BC5/Body_root_trailing_test.json" BrOp '{"obj":1,"k":3}' +run_compare "$BC4" "$BC5" "$BR_TESTS" +if [ "$RUN_RC" -eq 1 ] && grep -q "pairwiseEqual()" <<<"$RUN_OUT"; then + pass "O2: trailing body-root path violation still fails (exit 1)" +else + fail "O2: expected exit 1 with body-root violation; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test P: a waiver whose `paths` is a bare string (not an array) is an +# operator error — flatten would otherwise accept the string as one allowed +# path and silently suppress the enforcing rule (false-green, not a typo). +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs P)" +WV_TESTS="$TMP/P/wv-tests.json" +cat >"$WV_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Waiver type test", + "operation": "WvOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["memories"], "reason": "no shrink" }, + { "type": "pairwiseDeltaAllowed", "paths": "memories", "reason": "typo: bare string, must be rejected" } + ] + } +] +JSON +write_snapshot "$BC4/Waiver_type_test.json" WvOp '{"memories":[1,2,3]}' +write_snapshot "$BC5/Waiver_type_test.json" WvOp '{"memories":[]}' +run_compare "$BC4" "$BC5" "$WV_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "array of strings" <<<"$RUN_OUT"; then + pass "P: bare-string waiver paths fails with exit 2, no silent suppression" +else + fail "P: expected exit 2 with waiver type error; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test Q: a page object missing the documented keys (e.g. no body) is a +# structurally invalid snapshot — reads would return null and false-green. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs Q)" +write_snapshot "$BC4/Eq_order_test.json" EqOp '{"obj":{"a":1}}' +printf '{"operation":"EqOp","pages":[{"status":200}],"pages_count":1}' >"$BC5/Eq_order_test.json" +run_compare "$BC4" "$BC5" "$EQ_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "structurally invalid wire snapshot" <<<"$RUN_OUT"; then + pass "Q: page missing documented keys fails with exit 2" +else + fail "Q: expected exit 2 for body-less page; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test R: a waiver without a reason is rejected at runtime — schema.json +# requires reasons for accepted divergences, and this script also runs +# standalone (check-bc5-compat, scheduled workflow) with no schema step, +# so an unaudited waiver must not suppress enforcement. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs R)" +NR_TESTS="$TMP/R/nr-tests.json" +cat >"$NR_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Waiver reason test", + "operation": "WrOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["memories"], "reason": "no shrink" }, + { "type": "pairwiseDeltaAllowed", "paths": ["memories"] } + ] + } +] +JSON +write_snapshot "$BC4/Waiver_reason_test.json" WrOp '{"memories":[1,2,3]}' +write_snapshot "$BC5/Waiver_reason_test.json" WrOp '{"memories":[]}' +run_compare "$BC4" "$BC5" "$NR_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "non-empty 'reason'" <<<"$RUN_OUT"; then + pass "R: reason-less waiver fails with exit 2, no unaudited suppression" +else + fail "R: expected exit 2 with reason requirement; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test S: a snapshot whose recorded operation doesn't match the test's is an +# operator error — a stale/overwritten file with the right name must not be +# compared as this test's capture. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs S)" +write_snapshot "$BC4/Eq_order_test.json" EqOp '{"obj":{"a":1}}' +write_snapshot "$BC5/Eq_order_test.json" SomeOtherOp '{"obj":{"a":1}}' +run_compare "$BC4" "$BC5" "$EQ_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "stale or overwritten snapshot" <<<"$RUN_OUT"; then + pass "S: snapshot/test operation mismatch fails with exit 2" +else + fail "S: expected exit 2 with operation mismatch; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test T: a waiver with an empty (or missing) `paths` array is an operator +# error at runtime — schema.json's minItems:1 doesn't apply when the compare +# script runs standalone, and an empty waiver is always a fixture mistake. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs T)" +EW_TESTS="$TMP/T/ew-tests.json" +cat >"$EW_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Waiver empty paths test", + "operation": "EwOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseEqual", "paths": ["obj"], "reason": "must match" }, + { "type": "pairwiseDeltaAllowed", "paths": [], "reason": "empty waiver is a fixture mistake" } + ] + } +] +JSON +write_snapshot "$BC4/Waiver_empty_paths_test.json" EwOp '{"obj":1}' +write_snapshot "$BC5/Waiver_empty_paths_test.json" EwOp '{"obj":1}' +run_compare "$BC4" "$BC5" "$EW_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "non-empty array" <<<"$RUN_OUT"; then + pass "T: empty waiver paths fails with exit 2 at runtime" +else + fail "T: expected exit 2 with non-empty-paths error; got rc=$RUN_RC: $RUN_OUT" +fi + +# --------------------------------------------------------------------------- +# Test U: '[*]' anywhere other than the leading 'pages[*]' segment is an +# unsupported path — reject as a fixture mistake rather than streaming +# through jq with undefined comparison semantics. +# --------------------------------------------------------------------------- +read -r BC4 BC5 <<<"$(fresh_dirs U)" +UP_TESTS="$TMP/U/up-tests.json" +cat >"$UP_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Unsupported star test", + "operation": "UsOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["items[*].foo"], "reason": "undocumented star form" } + ] + } +] +JSON +write_snapshot "$BC4/Unsupported_star_test.json" UsOp '{"items":[{"foo":[1]}]}' +write_snapshot "$BC5/Unsupported_star_test.json" UsOp '{"items":[{"foo":[1]}]}' +run_compare "$BC4" "$BC5" "$UP_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "only supported as the leading" <<<"$RUN_OUT"; then + pass "U: non-pages '[*]' path fails with exit 2 as unsupported" +else + fail "U: expected exit 2 with unsupported-path error; got rc=$RUN_RC: $RUN_OUT" +fi + +# U2: a SECOND '[*]' after a valid leading 'pages[*]' is equally unsupported. +read -r BC4 BC5 <<<"$(fresh_dirs U2)" +UP2_TESTS="$TMP/U2/up2-tests.json" +cat >"$UP2_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Double star test", + "operation": "DsOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["pages[*].body.items[*]"], "reason": "second star is unsupported" } + ] + } +] +JSON +write_snapshot "$BC4/Double_star_test.json" DsOp '{"items":[[1]]}' +write_snapshot "$BC5/Double_star_test.json" DsOp '{"items":[[1]]}' +run_compare "$BC4" "$BC5" "$UP2_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "only supported as the leading" <<<"$RUN_OUT"; then + pass "U2: second '[*]' after pages[*] fails with exit 2 as unsupported" +else + fail "U2: expected exit 2 with unsupported-path error; got rc=$RUN_RC: $RUN_OUT" +fi + +# U3: an unsupported path stays a fixture error even when the same path is +# listed in pairwiseDeltaAllowed — the waiver skip must not bypass path +# validation, or a typo in a waived rule would permanently suppress +# enforcement without ever surfacing. +read -r BC4 BC5 <<<"$(fresh_dirs U3)" +UP3_TESTS="$TMP/U3/up3-tests.json" +cat >"$UP3_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Waived star test", + "operation": "WsOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["items[*].foo"], "reason": "undocumented star form" }, + { "type": "pairwiseDeltaAllowed", "paths": ["items[*].foo"], "reason": "waiver must not hide the unsupported path" } + ] + } +] +JSON +write_snapshot "$BC4/Waived_star_test.json" WsOp '{"items":[{"foo":[1]}]}' +write_snapshot "$BC5/Waived_star_test.json" WsOp '{"items":[{"foo":[1]}]}' +run_compare "$BC4" "$BC5" "$UP3_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "only supported as the leading" <<<"$RUN_OUT"; then + pass "U3: waived unsupported path still fails with exit 2" +else + fail "U3: expected exit 2 with unsupported-path error despite waiver; got rc=$RUN_RC: $RUN_OUT" +fi + +# Test V: a misspelled rule type is an operator error even when every path is +# waived — the type check must run before the waiver skip, or the fixture +# error exits clean and the (misspelled, so never-enforced) rule silently +# stops guarding anything. +read -r BC4 BC5 <<<"$(fresh_dirs V)" +VT_TESTS="$TMP/V/vt-tests.json" +cat >"$VT_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Misspelled type test", + "operation": "MtOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetAray", "paths": ["memories"], "reason": "typo'd rule type" }, + { "type": "pairwiseDeltaAllowed", "paths": ["memories"], "reason": "waiver must not hide the unknown rule type" } + ] + } +] +JSON +write_snapshot "$BC4/Misspelled_type_test.json" MtOp '{"memories":[1]}' +write_snapshot "$BC5/Misspelled_type_test.json" MtOp '{"memories":[]}' +run_compare "$BC4" "$BC5" "$VT_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "unknown pairwise rule type" <<<"$RUN_OUT"; then + pass "V: misspelled rule type fails with exit 2 despite fully-waived paths" +else + fail "V: expected exit 2 with unknown-rule-type error despite waiver; got rc=$RUN_RC: $RUN_OUT" +fi + +# Test W: a bare jq-stream typo like `items[]` (no '[*]', so it dodged the +# star guard) must be rejected as unsupported — unwrapped streams make the +# length comparison error falsy, silently skipping a real shrink violation +# (BC4 [1,2] vs BC5 [] would exit 0). +read -r BC4 BC5 <<<"$(fresh_dirs W)" +WS_TESTS="$TMP/W/ws-tests.json" +cat >"$WS_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Bare stream test", + "operation": "BsOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["items[]"], "reason": "bare [] is an unsupported jq stream" } + ] + } +] +JSON +write_snapshot "$BC4/Bare_stream_test.json" BsOp '{"items":[1,2]}' +write_snapshot "$BC5/Bare_stream_test.json" BsOp '{"items":[]}' +run_compare "$BC4" "$BC5" "$WS_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "brackets are only supported" <<<"$RUN_OUT"; then + pass "W: bare 'items[]' stream path fails with exit 2, no false-green" +else + fail "W: expected exit 2 with unsupported-brackets error; got rc=$RUN_RC: $RUN_OUT" +fi + +# Test X: a malformed bracket path that appears ONLY in a pairwiseDeltaAllowed +# waiver (no enforcing rule targets it) must still be a fixture error — it +# names a path no enforcing rule can ever use, so it would otherwise sit +# unnoticed while appearing to document a waiver. +read -r BC4 BC5 <<<"$(fresh_dirs X)" +XW_TESTS="$TMP/X/xw-tests.json" +cat >"$XW_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Waiver only bad path test", + "operation": "WbOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["items"], "reason": "valid enforcing rule" }, + { "type": "pairwiseDeltaAllowed", "paths": ["items[*].foo"], "reason": "typo'd waiver path that no enforcing rule targets" } + ] + } +] +JSON +write_snapshot "$BC4/Waiver_only_bad_path_test.json" WbOp '{"items":[1]}' +write_snapshot "$BC5/Waiver_only_bad_path_test.json" WbOp '{"items":[1]}' +run_compare "$BC4" "$BC5" "$XW_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -Eq "pairwiseDeltaAllowed.*brackets are only supported|brackets are only supported.*pairwiseDeltaAllowed" <<<"$RUN_OUT"; then + pass "X: malformed bracket path in a waiver-only entry fails with exit 2" +else + fail "X: expected exit 2 with waiver unsupported-path error; got rc=$RUN_RC: $RUN_OUT" +fi + +# Test Y: a bracketed path that never enters the page body (pages[0].items +# instead of pages[0].body.items) reads null on BOTH snapshots and would +# false-green a superset rule — reject it as a fixture mistake. +read -r BC4 BC5 <<<"$(fresh_dirs Y)" +YB_TESTS="$TMP/Y/yb-tests.json" +cat >"$YB_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Bodyless bracket path test", + "operation": "BbOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["pages[0].items"], "reason": "missing .body segment" } + ] + } +] +JSON +write_snapshot "$BC4/Bodyless_bracket_path_test.json" BbOp '{"items":[1,2]}' +write_snapshot "$BC5/Bodyless_bracket_path_test.json" BbOp '{"items":[]}' +run_compare "$BC4" "$BC5" "$YB_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "must enter the page body" <<<"$RUN_OUT"; then + pass "Y: bracket path skipping .body fails with exit 2, no false-green" +else + fail "Y: expected exit 2 with must-enter-body error; got rc=$RUN_RC: $RUN_OUT" +fi + +# Test Z: a snapshot whose page keys exist but carry the wrong TYPES +# (status as string, headers null) must fail structural validation — typed +# corruption would make read_value return nulls and false-green rules. +read -r BC4 BC5 <<<"$(fresh_dirs Z)" +write_snapshot "$BC4/$GMN_SAFE_NAME.json" GetMyNotifications '{"memories":[1]}' +mkdir -p "$BC5" +cat >"$BC5/$GMN_SAFE_NAME.json" <<'JSON' +{ + "operation": "GetMyNotifications", + "pages": [ + { "status": "200", "headers": null, "body": {"memories": []}, "bodyText": "{}", "url": "https://example.test" } + ], + "pages_count": 1 +} +JSON +run_compare "$BC4" "$BC5" "$GMN_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "structurally invalid wire snapshot" <<<"$RUN_OUT"; then + pass "Z: wrongly-typed page keys fail structural validation (exit 2)" +else + fail "Z: expected exit 2 for wrongly-typed page keys; got rc=$RUN_RC: $RUN_OUT" +fi + +# Test AA: a concrete pages[N] rule whose index is absent from BOTH captures +# is a fixture typo — reading it would compare null vs null and pass clean. +read -r BC4 BC5 <<<"$(fresh_dirs AA)" +AA_TESTS="$TMP/AA/aa-tests.json" +cat >"$AA_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Absent page index test", + "operation": "ApOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["pages[1].body.items"], "reason": "page 1 never captured" } + ] + } +] +JSON +write_snapshot "$BC4/Absent_page_index_test.json" ApOp '{"items":[1,2]}' +write_snapshot "$BC5/Absent_page_index_test.json" ApOp '{"items":[]}' +run_compare "$BC4" "$BC5" "$AA_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "targets page 1 but the .* snapshot has only" <<<"$RUN_OUT"; then + pass "AA: pages[1] against single-page captures fails with exit 2" +else + fail "AA: expected exit 2 with page-index error; got rc=$RUN_RC: $RUN_OUT" +fi + +# AB: page absent from ONE side only (BC4 captured 2 pages, BC5 only 1) is +# equally an error — a silent page-count regression must not read as null +# and pass the rule. +read -r BC4 BC5 <<<"$(fresh_dirs AB)" +AB_TESTS="$TMP/AB/ab-tests.json" +cat >"$AB_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "One sided page loss test", + "operation": "OpOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["pages[1].body.items"], "reason": "BC5 lost a page" } + ] + } +] +JSON +write_snapshot_2p "$BC4/One_sided_page_loss_test.json" OpOp '{"items":[1]}' '{"items":[2]}' +write_snapshot "$BC5/One_sided_page_loss_test.json" OpOp '{"items":[1]}' +run_compare "$BC4" "$BC5" "$AB_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "targets page 1 but the bc5 snapshot has only 1" <<<"$RUN_OUT"; then + pass "AB: one-sided page loss fails with exit 2, no null false-green" +else + fail "AB: expected exit 2 naming the bc5 snapshot; got rc=$RUN_RC: $RUN_OUT" +fi + +# Test AC: jq punctuation WITHOUT brackets (comma, space) must be rejected — +# the whitelist admits only dotted [A-Za-z0-9_] segments, so 'items, .pages' +# can't be interpolated into the jq program as raw syntax. +read -r BC4 BC5 <<<"$(fresh_dirs AC)" +AC_TESTS="$TMP/AC/ac-tests.json" +cat >"$AC_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Comma path test", + "operation": "CpOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["items, .pages"], "reason": "jq punctuation without brackets" } + ] + } +] +JSON +write_snapshot "$BC4/Comma_path_test.json" CpOp '{"items":[1,2]}' +write_snapshot "$BC5/Comma_path_test.json" CpOp '{"items":[]}' +run_compare "$BC4" "$BC5" "$AC_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "paths are dotted" <<<"$RUN_OUT"; then + pass "AC: jq punctuation without brackets fails with exit 2, no false-green" +else + fail "AC: expected exit 2 with dotted-segments error; got rc=$RUN_RC: $RUN_OUT" +fi + +# Test AD: hyphenated segments must be rejected — '.foo-bar' parses in jq as +# subtraction ('.foo - bar'), not a key lookup, so a hyphen-keyed rule would +# evaluate to garbage instead of reading the field. +read -r BC4 BC5 <<<"$(fresh_dirs AD)" +AD_TESTS="$TMP/AD/ad-tests.json" +cat >"$AD_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Hyphen path test", + "operation": "HpOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["bubble-ups"], "reason": "hyphen would parse as jq subtraction" } + ] + } +] +JSON +write_snapshot "$BC4/Hyphen_path_test.json" HpOp '{"bubble-ups":[1]}' +write_snapshot "$BC5/Hyphen_path_test.json" HpOp '{"bubble-ups":[]}' +run_compare "$BC4" "$BC5" "$AD_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "paths are dotted" <<<"$RUN_OUT"; then + pass "AD: hyphenated path fails with exit 2, no jq-subtraction misparse" +else + fail "AD: expected exit 2 with dotted-segments error; got rc=$RUN_RC: $RUN_OUT" +fi + +# Test AE: pairwiseEqual violations must NOT echo raw live values into the +# log (CI run logs are retained and readable) — only shape summaries. +read -r BC4 BC5 <<<"$(fresh_dirs AE)" +AE_TESTS="$TMP/AE/ae-tests.json" +cat >"$AE_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Redacted equal test", + "operation": "ReOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseEqual", "paths": ["secret_field"], "reason": "value content must stay out of logs" } + ] + } +] +JSON +write_snapshot "$BC4/Redacted_equal_test.json" ReOp '{"secret_field":"alice-private-note-bc4"}' +write_snapshot "$BC5/Redacted_equal_test.json" ReOp '{"secret_field":"alice-private-note-bc5"}' +run_compare "$BC4" "$BC5" "$AE_TESTS" +if [ "$RUN_RC" -eq 1 ] && grep -q "values differ" <<<"$RUN_OUT" && ! grep -q "alice-private-note" <<<"$RUN_OUT"; then + pass "AE: pairwiseEqual violation redacts raw values from output" +else + fail "AE: expected exit 1 with redacted shapes only; got rc=$RUN_RC: $RUN_OUT" +fi + +# Test AF: a shorthand path (normalizing to pages[0].body) against a +# multi-page snapshot is ambiguous — it would silently check only page 0. +# Explicit pages[N]/pages[*] syntax is required once either capture paginates. +read -r BC4 BC5 <<<"$(fresh_dirs AF)" +AF_TESTS="$TMP/AF/af-tests.json" +cat >"$AF_TESTS" <<'JSON' +[ + { + "mode": "live", + "name": "Shorthand multipage test", + "operation": "SmOp", + "method": "GET", + "path": "/x", + "liveAssertions": [{ "type": "liveCallSucceeds" }], + "pairwiseAssertions": [ + { "type": "pairwiseSupersetArray", "paths": ["items"], "reason": "shorthand against paginated capture" } + ] + } +] +JSON +write_snapshot_2p "$BC4/Shorthand_multipage_test.json" SmOp '{"items":[1]}' '{"items":[2]}' +write_snapshot "$BC5/Shorthand_multipage_test.json" SmOp '{"items":[1,2]}' +run_compare "$BC4" "$BC5" "$AF_TESTS" +if [ "$RUN_RC" -eq 2 ] && grep -q "is ambiguous" <<<"$RUN_OUT"; then + pass "AF: shorthand path against multi-page snapshot fails with exit 2" +else + fail "AF: expected exit 2 with ambiguity error; got rc=$RUN_RC: $RUN_OUT" +fi + +echo "" +if [ "$FAILURES" -ne 0 ]; then + echo "FAILED: $FAILURES test(s)" >&2 + exit 1 +fi +echo "All compare-canary-runs regression tests passed"