fix(0141): build and verify the Lambdas before every deploy

.github/workflows/ci.yml

@@ -101,6 +101,14 @@ jobs:
       # obviously missing. The portal changed that: its Vitest suite is the only
       # thing standing between a regression and `/api-tokens/`, and a suite that
       # runs solely on the author's machine guards nothing after the PR is open.
+      #
+      # It also runs the deploy-path guards of task 0141 — the infra project's
+      # `test` target is `tools/scripts/*.test.mjs`: the bootstrap verifier
+      # against stubbed, host-built, missing and duplicated artifacts, and
+      # `infra/Makefile` (every deploy able to ship a Lambda builds first, every
+      # per-stack deploy is `--exclusively`). As an Nx target and not a bare
+      # step so that `verify:staged` / `verify:push` run it too, on the pinned
+      # Node: `node --test <dir>` works on 26 and fails on the 22 CI uses.
       - run: npx nx run-many -t lint build typecheck test
 
       # Cheap counterpart to the rust job's build+verify+synth guard.
@@ -142,7 +150,7 @@ jobs:
       # rustc 1.98.0 (2026-08-18) began passing `-Wl,--fix-cortex-a53-843419`
       # on aarch64-unknown-linux-gnu. Zig's linker rejects it outright
       # ("error: unsupported linker arg"), so every aarch64 link fails and the
-      # `Build Lambda bootstraps` step dies on the first crate that links —
+      # `Build and verify Lambda bootstraps` step dies on the first crate that links —
       # `crc-fast`, pulled in by aws-smithy-checksums. Nothing in this repo
       # changed: PR #227 passed on 1.97.1 and PR #230 failed on 1.98.0 with the
       # identical cargo-lambda 1.9.1 / zig 0.16.0 pair.
@@ -199,59 +207,16 @@ jobs:
       # a bare `cargo lambda build` silently skips these bins and produces
       # only unrelated CLI binaries.
       #
-      # The list is read straight from the script in each step rather than
-      # passed between steps through `$GITHUB_OUTPUT`. The script refuses to
-      # emit an empty list, but that guarantee does not survive a step
-      # boundary: if the producing step were renamed or its id changed, the
-      # consumer's `${{ steps.… }}` would expand to nothing, the loop would
-      # run zero iterations and the guard would report success having checked
-      # nothing. Re-running a grep is cheaper than that failure mode. It also
-      # keeps workflow-expression text out of the shell body entirely.
-      - name: Build Lambda bootstraps
-        shell: bash
-        run: |
-          set -euo pipefail
-          args=()
-          while IFS= read -r name; do
-            [[ -z "$name" ]] && continue
-            args+=(-p "$name")
-          done < <(tools/scripts/lambda-assets.sh)
-          if [[ ${#args[@]} -eq 0 ]]; then
-            echo "::error::resolved zero Lambda assets; refusing to run a no-op build" >&2
-            exit 1
-          fi
-          echo "cargo lambda build --release --arm64 --features lambda ${args[*]}"
-          cargo lambda build --release --arm64 --features lambda "${args[@]}"
-
-      # Assert every bootstrap the CDK app references actually built. A bare
-      # "any bootstrap exists" glob would false-pass on the unrelated CLIs,
-      # so check each expected path explicitly.
-      - name: Verify Lambda artifacts
-        shell: bash
-        run: |
-          set -euo pipefail
-          echo "=== Lambda bootstrap binaries ==="
-          missing=0
-          checked=0
-          while IFS= read -r name; do
-            [[ -z "$name" ]] && continue
-            checked=$((checked + 1))
-            bin="target/lambda/${name}/bootstrap"
-            if [[ -x "$bin" ]]; then
-              echo "$(sha256sum "$bin") $(stat --format='%s bytes' "$bin")"
-            else
-              echo "::error::missing Lambda bootstrap: $bin"
-              missing=1
-            fi
-          done < <(tools/scripts/lambda-assets.sh)
-          # A loop over an empty list exits 0 having asserted nothing, which
-          # reads as success. Fail instead.
-          if [[ $checked -eq 0 ]]; then
-            echo "::error::verified 0 Lambda bootstraps; the guard passed vacuously" >&2
-            exit 1
-          fi
-          echo "verified ${checked} bootstrap(s)"
-          exit $missing
+      # The invocation and the artifact check live in ONE script, which is
+      # also what `make deploy-production*` runs before it ships a Lambda (task
+      # 0141). The workstation used to have no build step at all, and giving it
+      # a second copy of this one is how the build list and the verify list
+      # drifted in 0077. The script refuses an empty asset list, builds, then refuses
+      # any bootstrap that is missing, not an aarch64 ELF, or byte-identical to
+      # another asset's — a bare "any bootstrap exists" glob would false-pass
+      # on the unrelated CLIs.
+      - name: Build and verify Lambda bootstraps
+        run: tools/scripts/build-lambda-assets.sh
 
       # Final proof that the app the operator deploys actually synthesizes
       # with the assets this job just built. The build+verify steps above
@@ -276,7 +241,7 @@ jobs:
       # minutes today vs 5 ARM plus 1-2 on the synth runner), and does not
       # remove the duplicate TS build either. Closed won't-do; numbers in
       # lore/3-wiki/project/ci-pipeline.md. The real cost here is
-      # `Build Lambda bootstraps` above, at 3m24s = 67% of the job.
+      # `Build and verify Lambda bootstraps` above, at 3m24s = 67% of the job.
       - uses: actions/setup-node@v4
         with:
           node-version-file: .nvmrc

docs/runbooks/0072-current-prices-mv-rollout.md

@@ -409,17 +409,31 @@ The `api-handler` Lambda lives in the **Compute** stack.
 
 ```bash
 cd infra
-make diff-production          # read-only; review before deploying
+make diff-production          # builds the Lambdas first, then a read-only diff; review it
 make deploy-production-compute
 ```
 
-Two things to know before running this:
-
+Three things to know before running this:
+
+- **The Rust must be built, and the target now builds it.** `Code.fromAsset`
+  zips whatever is in `target/lambda/<name>/`; nothing in `cdk` compiles Rust.
+  On 2026-08-03 this step shipped a **stale `prices-api`** — green deploy, clean
+  S3Key diff, stub responses — and only step 7 caught it (task 0141). Since
+  0141 `deploy-production-compute` depends on `build-lambdas`
+  (`tools/scripts/build-lambda-assets.sh`): cargo rebuilds what is out of date,
+  then every bootstrap is refused unless it is a distinct aarch64 ELF.
+  `diff-production` depends on it too, so the diff you review is of the artifacts
+  the deploy will ship; the deploy's own build is then a ~1 s no-op.
 - **It also heals the 0132 CFN drift.** The 0132 egress fix was shipped by a
   surgical `aws lambda update-function-code` precisely to avoid deploying the
   then-unrolled 0072 read-API, which left CloudFormation believing the live
   processor still ran the old asset. This deploy reconciles that — the code it
   ships is the same code already running, so expect no behaviour change there.
+  ⚠️ That is only true of a `prices-ledger-processor` built from a tree that
+  **contains** the 0132 fix. A bootstrap predating it would have "healed" the
+  drift by silently reverting the 99.9% egress reduction, behind the same
+  clean-looking diff. On 2026-08-03 it was current by luck; since 0141 it is
+  current because the deploy builds it.
 - **It deploys every Lambda in the stack from the current tree**, not just the
   API. Confirm `develop` holds nothing else unrolled before running it.
 
@@ -430,6 +444,12 @@ curl -sS -H "x-api-key: $PRICES_API_KEY" \
   "https://<api-host>/production/v1/assets/native/price" | jq .
 ```
 
+🔴 **Do not use `GET /health` to verify a deploy.** It is a keyless API Gateway
+**mock** integration: it never invokes a Lambda, so it returns 200 with the
+handler stale, stubbed, or failing at init. Nor is a green `cdk deploy`
+evidence — it reports that CloudFormation accepted an asset, not what the asset
+is. Verify response **content**, as below.
+
 **The gate is `sources`, and only `sources`.** It must be a populated JSON
 **object** (not `{}`). A `{}` here while the CH columns are populated means the
 handler is still the stubbed build — re-check step 6 shipped.

docs/runbooks/deploy-ledger-processor.md

@@ -18,8 +18,12 @@ Two things you must understand up front:
 2. **CDK packages a pre-built binary.** The ComputeStack consumes
    `target/lambda/prices-ledger-processor/bootstrap` via `Code.fromAsset` — it
    does **not** compile Rust at synth time. So whatever bootstrap is sitting in
-   that path is exactly what ships. **If you skip the build step, you silently
-   redeploy the old binary.**
+   that path is exactly what ships. Until task 0141 that made a skipped build a
+   **silent redeploy of the old binary** (it happened, 2026-08-03). Now
+   `make deploy-production-compute` and `make deploy-production-eventbridge`
+   depend on `make build-lambdas`, which builds every Lambda the CDK app
+   references and refuses any bootstrap that is not a distinct aarch64 ELF. A raw
+   `npx cdk … deploy` has no such guard — deploy through `make`.
 
 ### ⚠️ This runbook covers ONE of the two Lambdas that decode ledger XDR
 
@@ -41,14 +45,9 @@ both deploys before you start. The steps below apply unchanged to the
 EventBridge stack — substitute the build and the deploy target:
 
 ```bash
-cargo lambda build -p asset-discovery --release --arm64 --features lambda
-cd infra && make deploy-production-eventbridge
+cd infra && make deploy-production-eventbridge   # builds the Lambdas first
 ```
 
-⚠️ `--features lambda` is mandatory here too: `asset-discovery`'s bin is
-`required-features = ["lambda"]`, so a build without it **silently skips the bin
-and produces no bootstrap** — the same trap as step 1 below.
-
 > First shipped this way in task 0066 (RustFunction adoption is a later
 > follow-up). The proto27 unfreeze (tasks 0091 → 0094) is the motivating case:
 > the xdr-27 decode fix (PR #104) only reaches the running Lambda once deployed
@@ -94,27 +93,44 @@ whether the binary you are about to ship can decode the ledgers it will be
 handed. See [Protocol-version lag](#protocol-version-lag--the-standing-check)
 below for what its three answers mean.
 
-### 1. Build the ARM64 bootstrap **with the new code**
-
-`--features lambda` is **mandatory**. The `prices-ledger-processor` bin is
-declared `required-features = ["lambda"]`, so a build without it silently skips
-the bin and produces no bootstrap.
+### 1. Build the ARM64 bootstraps **with the new code**
 
 ```bash
-cargo lambda build -p prices-ledger-processor --release --arm64 --features lambda
+cd infra && make build-lambdas
 ```
 
-→ writes `target/lambda/prices-ledger-processor/bootstrap`.
+This is the same script CI runs (`tools/scripts/build-lambda-assets.sh`): one
+`cargo lambda build --release --arm64 --features lambda -p …` over every asset
+the CDK app references, the list derived from the CDK source. Do not hand-type
+the cargo command instead — `--features lambda` is mandatory (each bin is
+`required-features = ["lambda"]`, so a build without it silently skips the bin
+and produces no bootstrap), and a single-crate build resolves cargo features
+differently from the group build, so it can yield a different binary from the one
+CI verified.
 
-### 2. Confirm the artifact is fresh and correct
+Step 4 runs this again by itself; doing it here first is what makes the diff in
+step 3 a diff of the artifacts that will ship — step 3 uses a raw `npx cdk diff`,
+which builds nothing (`make diff-production` does, but see step 3 for why not).
 
-```bash
-ls -l target/lambda/prices-ledger-processor/bootstrap    # mtime = seconds ago
-file target/lambda/prices-ledger-processor/bootstrap     # ELF 64-bit ... ARM aarch64
+Needs `cargo-lambda` and, on an x86 machine, `zig`. 🔴 Use the toolchain CI
+pins (`.github/workflows/ci.yml`: rustc 1.97.1, cargo-lambda 1.9.1) — rustc ≥
+1.98 fails every aarch64 link under zig with `unsupported linker arg`. That
+failure is loud; it cannot ship anything.
+
+### 2. Confirm the artifacts are correct
+
+The build ends by verifying every bootstrap and printing one line per asset:
+
+```
+<sha256>  prices-ledger-processor  14134024 bytes
+…
+verified 11 Lambda bootstrap(s)
 ```
 
-This check is your guard against Step 1 having failed or been skipped — never
-deploy on a stale/missing artifact.
+It exits non-zero, naming the asset, if a bootstrap is missing, is not an ELF
+(the 10-byte `#!/bin/sh` stubs found on 2026-08-12), is not aarch64, or is
+byte-identical to another asset's. Freshness is not something to eyeball from an
+mtime: cargo decided what needed rebuilding.
 
 ### 3. Preview the change (read-only, safe)
 
@@ -158,10 +174,21 @@ than trusting the diff alone.
 make deploy-production-compute
 ```
 
-`deploy-production-compute` deploys only the ComputeStack. `make
+`deploy-production-compute` builds the Lambdas, then deploys only the
+ComputeStack — every per-stack target passes `--exclusively`, so no dependency
+stack rides along (before task 0141 none did, and `deploy-production-apigateway`
+would have deployed Compute as a side effect). `make
 deploy-production` deploys _all_ stacks — avoid it unless you intend a full-app
-deploy. Override the asset path with `LEDGER_PROCESSOR_ASSET_DIR` if building
-elsewhere.
+deploy.
+
+🔴 Do **not** export `LEDGER_PROCESSOR_ASSET_DIR` (or any other `*_ASSET_DIR`)
+to deploy a build from elsewhere. CDK would package that directory while the
+build and its checks ran on `target/lambda/`; `make build-lambdas` refuses to
+run with one set, and so with `CARGO_TARGET_DIR` pointing outside the repo, for
+the same reason — what is verified must be what ships. A change spanning Compute
+**and** ApiGateway (a new route and its handler) needs
+`deploy-production-compute` first, then `deploy-production-apigateway`: neither
+target deploys the other any more.
 
 ### 5. Verify the new code is live
 

infra/Makefile

@@ -36,10 +36,29 @@ destroy-cicd: build
 # its own.
 build-production: build
 
+# `build` above compiles the CDK TypeScript ONLY. The Lambda code is whatever
+# sits in `../target/lambda/<name>/` when `Code.fromAsset` zips it — so a deploy
+# that did not build the Rust ships the last thing anyone happened to build, and
+# every signal says it worked: a clean S3Key change in `cdk diff`, a green
+# deploy, a passing `/health` (a keyless API Gateway MOCK — it never reaches a
+# Lambda and cannot be used to verify one). Task 0141: 2026-08-03 shipped a
+# stale `prices-api` this way; on 2026-08-12 the same directory held eleven
+# 10-byte `#!/bin/sh` stubs.
+#
+# So every target that can ship a Lambda depends on this. It is cargo that
+# decides freshness — a no-op when the artifacts are current — and the script
+# then refuses anything that is not a distinct aarch64 ELF per asset.
+build-lambdas:
+	../tools/scripts/build-lambda-assets.sh
+
 synth-production: build-production
 	npx cdk --app "$(PRODUCTION_APP)" synth
 
-diff-production: build-production
+# Depends on `build-lambdas` for the same reason the deploys do (task 0141): a
+# diff against stale artifacts that happen to match what is live shows no
+# Lambda change, and the deploy that follows then rebuilds and replaces them.
+# What you diff must be what you ship. ~1 s when the artifacts are current.
+diff-production: build-production build-lambdas
 	npx cdk --app "$(PRODUCTION_APP)" diff
 
 # The /api-docs-json stage-cache entry has a 3600s TTL, which is only safe
@@ -69,29 +88,35 @@ flush-production-cache:
 # So `make diff-production` before any production deploy, and read it for removals
 # rather than skimming it for additions. This applies to every deploy target below
 # too; it lives here because this is the one that deploys all stacks at once.
-deploy-production: build-production
+deploy-production: build-production build-lambdas
 	npx cdk --app "$(PRODUCTION_APP)" deploy --all --require-approval broadening
 	$(MAKE) flush-production-cache
 
 destroy-production: build-production
 	npx cdk --app "$(PRODUCTION_APP)" destroy --all --force
 
 deploy-production-secrets: build-production
-	npx cdk --app "$(PRODUCTION_APP)" deploy Prices-production-Secrets --require-approval broadening
+	npx cdk --app "$(PRODUCTION_APP)" deploy Prices-production-Secrets --exclusively --require-approval broadening
 
 destroy-production-secrets: build-production
 	npx cdk --app "$(PRODUCTION_APP)" destroy Prices-production-Secrets --force
 
 # Ships a new api-handler, so it ships a new OpenAPI document — flush too.
-deploy-production-compute: build-production
-	npx cdk --app "$(PRODUCTION_APP)" deploy Prices-production-Compute --require-approval broadening
+deploy-production-compute: build-production build-lambdas
+	npx cdk --app "$(PRODUCTION_APP)" deploy Prices-production-Compute --exclusively --require-approval broadening
 	$(MAKE) flush-production-cache
 
 destroy-production-compute: build-production
 	npx cdk --app "$(PRODUCTION_APP)" destroy Prices-production-Compute --force
 
+# `--exclusively` matters most here. ApiGateway imports the api-handler from
+# Compute, so without the flag this target deployed Compute too — from whatever
+# was in `target/lambda/` (task 0141, 2026-08-12). The price: a change that
+# spans both stacks (a new route AND its handler) needs
+# `deploy-production-compute` FIRST, then this. Deploy only this and the new
+# route proxies to the old handler behind a green deploy.
 deploy-production-apigateway: build-production
-	npx cdk --app "$(PRODUCTION_APP)" deploy Prices-production-ApiGateway --require-approval broadening
+	npx cdk --app "$(PRODUCTION_APP)" deploy Prices-production-ApiGateway --exclusively --require-approval broadening
 
 # Nothing imports this stack's exports since task 0195 retired
 # `PortalHostingStack` (which used the RestApi id for its execute-api origin),
@@ -171,14 +196,14 @@ sync-portal-explorer: build-portal-explorer verify-portal-guild
 	  --cache-control 'public, max-age=0, must-revalidate'
 	aws cloudfront create-invalidation --distribution-id $(EXPLORER_DISTRIBUTION_ID) --paths '/api/*'
 
-deploy-production-eventbridge: build-production
-	npx cdk --app "$(PRODUCTION_APP)" deploy Prices-production-EventBridge --require-approval broadening
+deploy-production-eventbridge: build-production build-lambdas
+	npx cdk --app "$(PRODUCTION_APP)" deploy Prices-production-EventBridge --exclusively --require-approval broadening
 
 destroy-production-eventbridge: build-production
 	npx cdk --app "$(PRODUCTION_APP)" destroy Prices-production-EventBridge --force
 
 deploy-production-observability: build-production
-	npx cdk --app "$(PRODUCTION_APP)" deploy Prices-production-Observability --require-approval broadening
+	npx cdk --app "$(PRODUCTION_APP)" deploy Prices-production-Observability --exclusively --require-approval broadening
 
 destroy-production-observability: build-production
 	npx cdk --app "$(PRODUCTION_APP)" destroy Prices-production-Observability --force

infra/README.md

@@ -181,7 +181,18 @@ make deploy-production-secrets  # single-stack scoped deploy
 ```
 
 Per-stack `deploy-production-{stack}` variants exist for every
-stack in the app — see `infra/Makefile`.
+stack in the app — see `infra/Makefile`. Each passes `--exclusively`: it
+deploys that stack and nothing it depends on.
+
+`make build` compiles the CDK TypeScript only. The Lambda code is whatever is
+in `../target/lambda/<name>/` at synth time, so every target that can ship a
+Lambda (`deploy-production`, `-compute`, `-eventbridge`) — and
+`diff-production`, so the diff is of what would ship — first runs
+`make build-lambdas` — `tools/scripts/build-lambda-assets.sh`, the same build
+CI runs, followed by a check that each bootstrap is a distinct aarch64 ELF
+(task 0141). Deploy through `make`; a raw `npx cdk deploy` ships whatever is on
+disk. And do not read `GET /health` as proof of a deploy: it is a gateway mock
+that never reaches a Lambda.
 
 ## Uploading the real mTLS PEMs
 

infra/package.json

@@ -24,7 +24,16 @@
     "tags": [
       "type:infra",
       "scope:infra"
-    ]
+    ],
+    "targets": {
+      "test": {
+        "command": "node --test \"tools/scripts/**/*.test.mjs\"",
+        "options": {
+          "cwd": "{workspaceRoot}"
+        },
+        "cache": false
+      }
+    }
   },
   "devDependencies": {
     "aws-cdk": "^2.1114.1"