diff --git a/.github/workflows/gate-concurrency-evidence.yml b/.github/workflows/gate-concurrency-evidence.yml deleted file mode 100644 index 958f25c58..000000000 --- a/.github/workflows/gate-concurrency-evidence.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: gate-concurrency-evidence - -# Replays the archived memory-default concurrency evidence. -# -# This is a conditional evidence suite, not part of the ordinary -# reference-implementation test plan: it runs only when the archive, its -# summary, the receipt builder/verifier, the replay itself, or the parser -# dependencies those two rely on change. There is deliberately no schedule -- -# archived bytes do not change on their own, so a timer would only add noise. -# -# The replay is also runnable by hand from reference-implementation/: -# node --test --experimental-strip-types scripts/evidence/gate-concurrency-receipts.test.ts - -on: - pull_request: - paths: - - .github/workflows/gate-concurrency-evidence.yml - - reference-implementation/docs/gate-concurrency.md - - reference-implementation/docs/receipts/gate-concurrency-20260903.summary.json - - reference-implementation/docs/receipts/gate-concurrency-20260903.tar.gz - - reference-implementation/scripts/evidence/gate-concurrency-receipts.test.ts - - reference-implementation/scripts/gate-concurrency-receipt.test.ts - - reference-implementation/scripts/gate-concurrency-receipt.ts - # Parser dependencies of the builder/verifier: a change in either can - # alter re-derived counts or digests without touching the files above. - - reference-implementation/scripts/test-accounting/inventory.ts - - reference-implementation/scripts/test-accounting/receipt.ts - merge_group: {} - workflow_dispatch: {} - -concurrency: - group: gate-concurrency-evidence-${{ github.ref }} - cancel-in-progress: true - -jobs: - replay: - name: replay archived gate concurrency evidence - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: "22.23.1" - - - name: Verify the receipt builder and verifier - working-directory: reference-implementation - run: node --test --experimental-strip-types scripts/gate-concurrency-receipt.test.ts - - - name: Replay the archived evidence - working-directory: reference-implementation - run: node --test --experimental-strip-types scripts/evidence/gate-concurrency-receipts.test.ts diff --git a/reference-implementation/docs/gate-concurrency.md b/reference-implementation/docs/gate-concurrency.md index b0133bd03..b902d40d3 100644 --- a/reference-implementation/docs/gate-concurrency.md +++ b/reference-implementation/docs/gate-concurrency.md @@ -26,61 +26,38 @@ otherwise-independent file workers can contend for the same restore resource. Raising concurrency for PostgreSQL needs its own restore-aware measurement on a PostgreSQL host; a memory-profile result is not authority for it. -## Archived memory-default measurements - -Two memory-default runs of the same tree and selection, one at cap 2 and one at -cap 8, are archived with this document: - -- `receipts/gate-concurrency-20260903.tar.gz` — the raw receipts and - transcripts, four members, byte-for-byte as recorded -- `receipts/gate-concurrency-20260903.summary.json` — a readable pairing of the - two runs - -Read a member without unpacking the archive: - -```sh -tar -xzOf docs/receipts/gate-concurrency-20260903.tar.gz \ - gate-concurrency-memory-cap-8.receipt.json -``` - -Replay the archived pair, which re-derives each receipt's counts, failure names -and selection digests from its own archived raw output: - -```sh -node --test --experimental-strip-types scripts/evidence/gate-concurrency-receipts.test.ts -``` - -## What the receipts establish - -Three different things are worth keeping separate: - -- **Recorded provenance.** Git head, Node version, profile and source-tree - digest are values the measuring process wrote down. Nothing here authenticates - the host or the toolchain; matching digests and paired metadata do not make - recorded provenance independently verified. -- **Digest binding.** Each receipt's digests bind its transcript, selected-file - list and selection manifest. This shows the bytes were not edited after - recording. -- **Re-derived outcomes.** The counts and failure identities are recomputed from - the raw structured output the transcript carries, so a forged count or a - renamed failure is rejected even when every digest still matches. - -`counts.completed_files` is a legacy field derived from the exit code, not an -observed completion count. It is 0 on both archived runs because both exited -non-zero. A real per-file completion claim needs raw file-outcome events, which -this schema does not carry. - -## What the pair observed - -On the archived Node 22.23.1 runs, both caps selected 1,033 files and produced -6,961 assertions: 6,335 passed, 396 failed and 230 skipped, with the same 396 -failure identities and exit code 1 in both runs. The cap-2 receipt records -352.198 seconds; the cap-8 receipt records 141.066 seconds. - -This is **failure-set equality for this pair**, on one host, with both runs -failing. Two runs that fail identically say nothing about whether either cap is -safe, and none of it is a green-suite result. The failures are retained as -evidence rather than hidden. +## The one-off measurement behind the cap + +The cap was set after a single pair of memory-default runs on 2026-09-03: the +same tree and the same selection, once at cap 2 and once at cap 8, on one host +at Node 22.23.1, git head `eb6a890d`. + +| | cap 2 | cap 8 | +| --- | --- | --- | +| Elapsed | 352.198 s | 141.066 s | +| Selected files | 1,033 | 1,033 | +| Assertions | 6,961 | 6,961 | +| Passed / failed / skipped | 6,335 / 396 / 230 | 6,335 / 396 / 230 | +| Failure identities | 396 | 396 (same set) | +| Exit code | 1 | 1 | + +Both runs failed, and they failed on the identical set of 396 assertions. That +is **failure-set equality for one pair on one host** — it says the cap did not +change which assertions failed, and nothing more. It is not a green-suite +result, and two equally failing runs are not evidence that either cap is safe. + +The raw receipts and transcripts for that pair are not retained. They were a +snapshot of one day's tree, they went stale the moment the tree moved, and +re-verifying a frozen archive proves only that the archive is unchanged. Going +forward, **the evidence is the ongoing CI runs at the current cap**: every +`reference-implementation` CI run exercises the default cap against the tree as +it actually is, which is the claim worth holding. + +Reading any run's counts, then or now: `completed_files` is derived from the +exit code (`run-tests.ts` writes `failed ? 0 : results.length`), not from an +observed per-file completion count, so it is 0 for any run that exits non-zero. +A real per-file completion claim needs raw file-outcome events, which the +receipt schema does not carry. ## Operational use diff --git a/reference-implementation/docs/receipts/gate-concurrency-20260903.summary.json b/reference-implementation/docs/receipts/gate-concurrency-20260903.summary.json deleted file mode 100644 index 8a4e749fa..000000000 --- a/reference-implementation/docs/receipts/gate-concurrency-20260903.summary.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "archive": { - "bytes": 444700, - "member_names": [ - "gate-concurrency-memory-cap-2.receipt.json", - "gate-concurrency-memory-cap-2.transcript", - "gate-concurrency-memory-cap-8.receipt.json", - "gate-concurrency-memory-cap-8.transcript" - ], - "name": "gate-concurrency-20260903.tar.gz", - "sha256": "39518957615d6fa36b60204c5fdbe928c045f42805942b0fd09da64f81659bf2" - }, - "note": "Readable pairing of two memory-default concurrency measurements. The raw receipts and transcripts are the evidence; this file is a summary of them. Full selected-file and failure-name lists stay in the archive and are not duplicated here.", - "pair_comparison": { - "both_exit_code": 1, - "comment": "Failure-set equality for this pair on this host. Both runs exited 1; equal failing runs are not evidence that either cap is safe.", - "same_assertion_counts": true, - "same_failure_identities": true, - "same_selected_files": true - }, - "recorded_provenance": { - "comment": "Recorded by the measuring process and bound by the receipt digests. Digest binding shows the bytes were not edited after recording; it does not independently authenticate the host or toolchain.", - "git_head": "eb6a890d57658c14aef706bdbe4f756b14238d21", - "node_version": "v22.23.1", - "profile": "memory-default", - "source_tree_sha256": "301a1138c7792be483917dc6ef7ffcab3e72fe62550d0f0b52cfaac26e8e6e4b" - }, - "runs": { - "2": { - "cap": 2, - "counts": { - "assertions": 6961, - "completed_files_legacy_exit_derived": 0, - "failed": 396, - "passed": 6335, - "planned_files": 1033, - "skipped": 230 - }, - "elapsed_seconds": 352.198, - "exit_code": 1, - "failure_identity_count": 396, - "members": { - "receipt": { - "bytes": 113305, - "name": "gate-concurrency-memory-cap-2.receipt.json", - "sha256": "89cff256c6c3b98380738bb822ac478e667061e2fd940c956c653c89aeea1113" - }, - "transcript": { - "bytes": 1962200, - "name": "gate-concurrency-memory-cap-2.transcript", - "sha256": "99211f3230d277e3a2144c19e9e0c901c19158a7ba3a227ec6e9f6ceb25c52ea" - } - }, - "selected_file_count": 1033 - }, - "8": { - "cap": 8, - "counts": { - "assertions": 6961, - "completed_files_legacy_exit_derived": 0, - "failed": 396, - "passed": 6335, - "planned_files": 1033, - "skipped": 230 - }, - "elapsed_seconds": 141.066, - "exit_code": 1, - "failure_identity_count": 396, - "members": { - "receipt": { - "bytes": 113305, - "name": "gate-concurrency-memory-cap-8.receipt.json", - "sha256": "9e85b032e7d131abffe432ac8332e9306dd731a8c71abc3070c37a61f80e9367" - }, - "transcript": { - "bytes": 1962200, - "name": "gate-concurrency-memory-cap-8.transcript", - "sha256": "94984add558328d9f5387caf12890d6f74c29579b73735edb9dbb1db7b1dd3e0" - } - }, - "selected_file_count": 1033 - } - }, - "schema": "pdpp.gate-concurrency-evidence-summary/v1", - "selection_identity": { - "identical_across_caps": true, - "selected_files_sha256": "d9835bfd772cce01504ad744b7d584345493778662840e44e3f91c9601858822", - "selection_manifest_sha256": "ce0fcb99c4678bb977b1842df32fa6aee47a5979698d3bf9270b7a1eab460bd9" - } -} diff --git a/reference-implementation/docs/receipts/gate-concurrency-20260903.tar.gz b/reference-implementation/docs/receipts/gate-concurrency-20260903.tar.gz deleted file mode 100644 index e969733e1..000000000 Binary files a/reference-implementation/docs/receipts/gate-concurrency-20260903.tar.gz and /dev/null differ diff --git a/reference-implementation/scripts/evidence/gate-concurrency-receipts.test.ts b/reference-implementation/scripts/evidence/gate-concurrency-receipts.test.ts deleted file mode 100644 index 4448b7fe3..000000000 --- a/reference-implementation/scripts/evidence/gate-concurrency-receipts.test.ts +++ /dev/null @@ -1,371 +0,0 @@ -// Copyright The PDP-Connect Contributors -// SPDX-License-Identifier: Apache-2.0 - -// Historical evidence replay for the archived memory-default concurrency pair. -// -// This file is not part of the ordinary backend test plan: it is a separately -// owned conditional evidence suite, run when the archive, its summary, the -// receipt verifier/builder, their parser dependencies -// (test-accounting/receipt.ts, test-accounting/inventory.ts) or this replay -// change. Run it directly with: -// -// node --test --experimental-strip-types scripts/evidence/gate-concurrency-receipts.test.ts -// -// It lives under scripts/evidence/ so the runner's non-recursive scripts/ -// discovery does not pull it into every gate run. - -import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { readFileSync } from "node:fs"; -import test from "node:test"; -import { fileURLToPath } from "node:url"; -import { - assertGateConcurrencyReceipt, - buildGateConcurrencyReceipt, - failureIdentities, - type GateConcurrencyReceipt, - transcriptOutput, -} from "../gate-concurrency-receipt.ts"; - -const ARCHIVE = fileURLToPath( - new URL( - "../../docs/receipts/gate-concurrency-20260903.tar.gz", - import.meta.url, - ), -); -const SUMMARY = fileURLToPath( - new URL( - "../../docs/receipts/gate-concurrency-20260903.summary.json", - import.meta.url, - ), -); - -interface EvidenceSummary { - archive: { member_names: string[]; name: string; sha256: string }; - pair_comparison: { - both_exit_code: number; - same_assertion_counts: boolean; - same_failure_identities: boolean; - same_selected_files: boolean; - }; - recorded_provenance: { - git_head: string; - node_version: string; - profile: string; - source_tree_sha256: string; - }; - runs: Record< - string, - { - cap: number; - counts: { - assertions: number; - failed: number; - passed: number; - planned_files: number; - skipped: number; - }; - elapsed_seconds: number; - exit_code: number; - failure_identity_count: number; - members: { - receipt: { name: string; sha256: string }; - transcript: { name: string; sha256: string }; - }; - selected_file_count: number; - } - >; - selection_identity: { - selected_files_sha256: string; - selection_manifest_sha256: string; - }; -} - -/** Reads one archive member without unpacking the archive to disk. */ -function readMember(name: string): string { - return execFileSync("tar", ["-xzOf", ARCHIVE, name], { - encoding: "utf8", - maxBuffer: 64 * 1024 * 1024, - }); -} - -function readArchivedPair(cap: 2 | 8): { - receipt: GateConcurrencyReceipt; - transcript: string; -} { - const prefix = `gate-concurrency-memory-cap-${cap}`; - return { - receipt: JSON.parse( - readMember(`${prefix}.receipt.json`), - ) as GateConcurrencyReceipt, - transcript: readMember(`${prefix}.transcript`), - }; -} - -function readSummary(): EvidenceSummary { - return JSON.parse(readFileSync(SUMMARY, "utf8")) as EvidenceSummary; -} - -function comparableCounts(receipt: GateConcurrencyReceipt) { - const { - assertions, - completed_files, - failed, - passed, - planned_files, - skip_reasons, - skipped, - } = receipt.counts; - return { - assertions, - completed_files, - failed, - passed, - planned_files, - skip_reasons, - skipped, - }; -} - -test("archived cap-2 and cap-8 receipts retain an equivalent memory-default result", () => { - const capTwo = readArchivedPair(2); - const capEight = readArchivedPair(8); - - assertGateConcurrencyReceipt(capTwo.receipt, capTwo.transcript); - assertGateConcurrencyReceipt(capEight.receipt, capEight.transcript); - assert.equal(capTwo.receipt.cap, 2); - assert.equal(capEight.receipt.cap, 8); - assert.deepEqual( - comparableCounts(capTwo.receipt), - comparableCounts(capEight.receipt), - ); - assert.deepEqual( - capTwo.receipt.failure_identities, - capEight.receipt.failure_identities, - ); - assert.deepEqual( - capTwo.receipt.selected_files, - capEight.receipt.selected_files, - ); - assert.equal(capTwo.receipt.git_head, capEight.receipt.git_head); - assert.equal(capTwo.receipt.node_version, capEight.receipt.node_version); - assert.equal( - capTwo.receipt.selection_manifest_sha256, - capEight.receipt.selection_manifest_sha256, - ); - assert.equal( - capTwo.receipt.source_tree_sha256, - capEight.receipt.source_tree_sha256, - ); - assert.equal(capTwo.receipt.exit_code, 1); - assert.equal(capEight.receipt.exit_code, 1); - assert.ok( - Date.parse(capTwo.receipt.ended_at) > Date.parse(capTwo.receipt.started_at), - ); - assert.ok( - Date.parse(capEight.receipt.ended_at) > - Date.parse(capEight.receipt.started_at), - ); -}); - -// The builder's live consumer. Rebuilding each receipt from its own archived -// raw output is what makes the archived counts, failure names and selection -// digests re-derived claims instead of self-reported ones. -test("rebuilding each archived receipt from its raw output reproduces the recorded claims", () => { - for (const cap of [2, 8] as const) { - const { receipt, transcript } = readArchivedPair(cap); - const output = transcriptOutput(transcript); - const rebuilt = buildGateConcurrencyReceipt({ - cap: receipt.cap, - endedAt: receipt.ended_at, - exitCode: receipt.exit_code, - gitHeadSha: receipt.git_head, - output, - selectedFiles: receipt.selected_files, - sourceTreeSha256: receipt.source_tree_sha256, - startedAt: receipt.started_at, - transcript, - }); - - // Compare the rederived fields only. node_version and transcript come from - // this replay process, not from the original measurement, so the whole - // rebuilt receipt is deliberately not compared. - for (const field of [ - "assertions", - "passed", - "failed", - "skipped", - "planned_files", - ] as const) { - assert.equal( - rebuilt.counts[field], - receipt.counts[field], - `cap ${cap} ${field}`, - ); - } - assert.deepEqual( - rebuilt.counts.skip_reasons, - receipt.counts.skip_reasons, - `cap ${cap} skip reasons`, - ); - assert.deepEqual( - rebuilt.failure_identities, - receipt.failure_identities, - `cap ${cap} failure identities`, - ); - assert.equal( - rebuilt.selected_files_sha256, - receipt.selected_files_sha256, - `cap ${cap} selected files digest`, - ); - assert.equal( - rebuilt.selection_manifest_sha256, - receipt.selection_manifest_sha256, - `cap ${cap} selection manifest digest`, - ); - assert.equal( - rebuilt.transcript_sha256, - receipt.transcript_sha256, - `cap ${cap} transcript digest`, - ); - assert.equal(rebuilt.exit_code, receipt.exit_code, `cap ${cap} exit code`); - assert.deepEqual( - failureIdentities(output), - receipt.failure_identities, - `cap ${cap} raw failure identities`, - ); - } -}); - -test("forged counts and failure names in an archived receipt are rejected", () => { - const { receipt, transcript } = readArchivedPair(8); - - const forgedCounts = { - ...receipt, - counts: { ...receipt.counts, failed: receipt.counts.failed - 1 }, - }; - assert.throws( - () => assertGateConcurrencyReceipt(forgedCounts, transcript), - /count does not match/, - ); - - const renamed = [...receipt.failure_identities]; - renamed[0] = "a failure that was never recorded"; - const forgedNames = { ...receipt, failure_identities: renamed }; - assert.throws( - () => assertGateConcurrencyReceipt(forgedNames, transcript), - /failure identities do not match the transcript output/, - ); - - const dropped = { - ...receipt, - failure_identities: receipt.failure_identities.slice(1), - }; - assert.throws( - () => assertGateConcurrencyReceipt(dropped, transcript), - /failure identity count does not match the transcript output/, - ); -}); - -test("corrupting the archived raw bytes is rejected", () => { - const { receipt, transcript } = readArchivedPair(2); - const [firstFailure] = receipt.failure_identities; - assert.ok( - firstFailure, - "the archived pair records at least one failure identity", - ); - - const corrupted = transcript.replace( - firstFailure, - "a name that is not in the receipt", - ); - assert.notEqual( - corrupted, - transcript, - "the corruption changed the raw bytes", - ); - assert.throws( - () => assertGateConcurrencyReceipt(receipt, corrupted), - /transcript digest does not match/, - ); -}); - -test("the readable summary agrees with the archived raw bytes", () => { - const summary = readSummary(); - const capTwo = readArchivedPair(2); - const capEight = readArchivedPair(8); - - assert.equal(summary.archive.name, "gate-concurrency-20260903.tar.gz"); - assert.deepEqual(summary.archive.member_names.toSorted(), [ - "gate-concurrency-memory-cap-2.receipt.json", - "gate-concurrency-memory-cap-2.transcript", - "gate-concurrency-memory-cap-8.receipt.json", - "gate-concurrency-memory-cap-8.transcript", - ]); - assert.equal(summary.recorded_provenance.git_head, capTwo.receipt.git_head); - assert.equal( - summary.recorded_provenance.node_version, - capTwo.receipt.node_version, - ); - assert.equal(summary.recorded_provenance.profile, capTwo.receipt.profile); - assert.equal( - summary.recorded_provenance.source_tree_sha256, - capTwo.receipt.source_tree_sha256, - ); - assert.equal( - summary.selection_identity.selection_manifest_sha256, - capTwo.receipt.selection_manifest_sha256, - ); - assert.equal( - summary.selection_identity.selected_files_sha256, - capTwo.receipt.selected_files_sha256, - ); - assert.equal(summary.pair_comparison.both_exit_code, 1); - - for (const [cap, pair] of [ - [2, capTwo], - [8, capEight], - ] as const) { - const run = summary.runs[String(cap)]; - assert.ok(run, `summary records cap ${cap}`); - assert.equal(run.cap, pair.receipt.cap); - assert.equal(run.exit_code, pair.receipt.exit_code); - assert.equal(run.selected_file_count, pair.receipt.selected_files.length); - assert.equal( - run.failure_identity_count, - pair.receipt.failure_identities.length, - ); - for (const field of [ - "assertions", - "passed", - "failed", - "skipped", - "planned_files", - ] as const) { - assert.equal( - run.counts[field], - pair.receipt.counts[field], - `summary cap ${cap} ${field}`, - ); - } - const elapsed = - (Date.parse(pair.receipt.ended_at) - - Date.parse(pair.receipt.started_at)) / - 1000; - assert.equal( - run.elapsed_seconds, - elapsed, - `summary cap ${cap} elapsed seconds`, - ); - } - - // The summary points at the archive; it must not become a second copy of the - // full selected-file or failure-name lists. - const summaryText = readFileSync(SUMMARY, "utf8"); - for (const identity of capEight.receipt.failure_identities.slice(0, 5)) { - assert.ok( - !summaryText.includes(identity), - "the summary does not duplicate failure names", - ); - } -});