diff --git a/coordinator-rs/.gitignore b/coordinator-rs/.gitignore new file mode 100644 index 0000000000..5e7da4b0b2 --- /dev/null +++ b/coordinator-rs/.gitignore @@ -0,0 +1,3 @@ +/target/ +/artifacts/ +__pycache__/ diff --git a/coordinator-rs/Cargo.lock b/coordinator-rs/Cargo.lock new file mode 100644 index 0000000000..8ddc1a57fa --- /dev/null +++ b/coordinator-rs/Cargo.lock @@ -0,0 +1,1095 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libredox" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" +dependencies = [ + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "polis-coordinator" +version = "0.1.0" +dependencies = [ + "anyhow", + "postgres", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "postgres" +version = "0.19.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ad20e0aa0b24f5a394eab4f78c781d248982b22b25cecc7e3aa46a681605bd" +dependencies = [ + "bytes", + "fallible-iterator", + "futures-util", + "log", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", + "serde_core", + "serde_json", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinyvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "uuid" +version = "1.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +dependencies = [ + "getrandom", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.5", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/coordinator-rs/Cargo.toml b/coordinator-rs/Cargo.toml new file mode 100644 index 0000000000..0ce79640dc --- /dev/null +++ b/coordinator-rs/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "polis-coordinator" +version = "0.1.0" +edition = "2024" +rust-version = "1.88" +publish = false + +[features] +default = [] +fault-injection = [] + +[dependencies] +anyhow = "1" +postgres = { version = "0.19", features = ["with-serde_json-1"] } +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["float_roundtrip"] } +sha2 = "0.10" +tempfile = "3" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] } +uuid = { version = "1", features = ["v4"] } + +[lints.clippy] +unwrap_used = "deny" +expect_used = "deny" diff --git a/coordinator-rs/README.md b/coordinator-rs/README.md new file mode 100644 index 0000000000..cd318a1602 --- /dev/null +++ b/coordinator-rs/README.md @@ -0,0 +1,438 @@ +# P-026 Rust coordinator experiment + +Experimental crate on a branch; not a deployment or merge proposal. No math +algorithms are changed. `polis-engine/1` is implemented as a **candidate-profile**, not a claim of +full contract certification. See the P-026 report for executed coverage and gaps. + +## Layout + +- `src/store.rs`: `ResultsStore` trait and PostgreSQL v0, coherent joined reads, + expected-tick conflicts, lease fencing, JSONB integrity digests, atomic publication. +- `src/lease.rs`: the three distinguishable ownership outcomes and the + conditional heartbeat renewal that keeps a long compute inside its lease. +- `src/ordering.rs`: the declared `polis-order/1` source normalization and its + `equal_time_census`; the coordinator's ORDER BY is built from it. +- `src/cache.rs`: bounded warm bundle cache with the LRU eviction stage. +- `src/coordinator.rs`: complete source snapshot reconciliation, durable keyset + cursor, bounded failure backoff, one active conversation and worker process. +- `src/probe.rs`: the cheap per-conversation change probe, the durable per-zid + reconciliation cursor that bounds how long it may be trusted, and the CO01/CO06 + backlog and scan-age aggregates. +- `src/metrics.rs`: the declared metric catalog, the pluggable sink and the + CloudWatch Embedded Metric Format records in namespace `Polis/Math`. +- `src/engine.rs`, `src/wire.rs`: bounded JSONL subprocess client, strict parsing, + immutable file descriptors, identity/checkpoint validation, rebuild-prefix lifecycle. +- `src/reader.rs`: metadata-first keyset sweep plus paginated trailing window. +- `tools/node_reader.cjs`: the D4 harness that loads the **real** server modules + `server/src/utils/pca.ts` and `server/src/utils/participants.ts` in one Node + process and reports the bytes they serve for each namespace. +- `tools/node_route_probe.cjs`: the real `handle_GET_math_pca2` mounted on a real + Express app over loopback HTTP, from Astra's round-2 review probe. Only the + parameter middleware is synthetic. It is what shows that the route serves a + committed generation of zero with 200 and ETag `"0"`, and a matching + conditional request 304, while `getPca(zid, undefined)` — a different caller + argument — misses it on a cold cache. +- `src/fault.rs`: external arm/ack/release file barriers, debug feature only. +- `migration.sql`: additive prototype metadata/leases/sequence; explicit command. +- `../delphi/polismath/engine_adapter.py`: Conversation lifecycle and existing row + derivation, with explicit empty serialization and snapshot un-moderation handling. +- `../delphi/tests/coordinator/`: process-neutral recovery/equivalence tests using + migrated PostgreSQL, pinned independent fold and R09 mapping assertions. + +The coordinator rebuilds from persisted warm fields and the entire authoritative +prefix for every changed source snapshot. This resets the smoother as the reference +poller does on restart. There is still no warm *worker* (Conversation) cache, so +this is not uninterrupted-warm equivalence and the live comparator uses exactly +this restart schedule. There is a bounded warm **bundle** cache +(`P026_CACHE_CAP`, default 16, 0 disables): the last coherent published bundle of +an unchanged conversation, so a quiet pass does not re-read the three results +tables. Lookup and LRU touch are one operation, so an eviction can never +interleave between them. A resident entry is **never** treated as evidence about +the durable store. On the fast path every hit re-verifies companion presence, +every companion's generation and the committed checkpoint identity against the +database (`resident_is_intact`, metadata and `input_checkpoint` only, no payload +column). That is cheap and it closes the deleted-companion case, but it cannot +see a mutated payload, so it is explicitly not complete integrity +reconciliation: whenever the reconciliation ceiling expires the authoritative +path re-reads and re-hashes the persisted generation with `load_current` and the +resident bundle is discarded. Payload corruption that leaves metadata intact therefore +becomes eligible for repair once the ceiling elapses, instead of surviving behind +a warm cache indefinitely. The ceiling is an eligibility threshold, not a measured +deadline: page traversal, backoff, lease waits and compute all sit between +eligibility and the repair, and no service budget has been measured here. Eviction is the CO07 +`cache_eviction_contends_with_same_zid_update` stage; that stage is the bounded +Bundle-cache profile only, not a warm-worker, four-worker or Node cache profile. + +**Resident size after S1.** A resident `Bundle` now holds both the parsed JSONB +values and the admitted original bytes (`Payloads::originals`), because exact +byte replay and the decoded-correspondence check both need the originals. Each +entry therefore costs roughly the original payload bytes *plus* their parsed +`serde_json::Value` representation — about twice what it cost before S1, at the +same `P026_CACHE_CAP`. The bound is still `cap ×` one generation, and eviction is +unchanged, but the earlier bounded-memory statement was measured against the old +per-entry constant: halve `P026_CACHE_CAP` to hold the previous ceiling. No new +measurement has been taken here. Relatedly, `read` and `publish-fixture` now +serialize originals as JSON integer arrays, so CLI stdout grows several-fold per +payload byte; that is a fixture-scale format, not an interface to script against. + +## Incremental discovery, and why it is only a filter + +Every pass still visits every admitted conversation independently of timestamps. +What changed is what a visit costs. Before reading the full vote/comment/ +participant history, the coordinator takes a cheap single-statement aggregate +probe (`src/probe.rs`) and compares it with the probe recorded, in +`coordinator_reconciliation`, immediately **before** the authoritative snapshot +that certified the published generation. A row committing between the probe and +the snapshot therefore changes the *next* probe; it cannot be swallowed. + +`reconciled_at` is the probe's own database timestamp, taken in the same +statement and therefore *before* the source read — not the time publication +finished. A compute that outlasts the ceiling cannot reset the advertised source +age and buy another fast-path interval. + +The probe can only skip a read. It never authorises a rebuild, and it is trusted +only while that conversation's last authoritative reconciliation is younger than +`P026_RECONCILE_SECONDS` (default 3600). That interval is an eligibility +threshold, not a proven end-to-end repair deadline: page traversal, backoff, +lease waits and compute all add latency, and no pass or service budget has been +measured here. Count and max are hints: a change that +leaves every aggregate identical really is invisible to them, and +`test_the_aggregate_probe_is_weak_but_the_reconciliation_ceiling_repairs_it` +stages exactly such a change to prove both halves. `P026_INCREMENTAL=0` disables +the fast path entirely and restores the unconditional full sweep. The probe also +carries `ordering::algorithm_digest`, so changing the declared normalization or +the storage agree-convention constant invalidates every conversation even though +no source row moved. + +`OldestReconciliationAgeSeconds` is what makes this auditable: the fast path is +sound only while that age stays bounded, so the metric is part of the mechanism +rather than decoration. It is not publication lag. + +## Metrics + +`src/metrics.rs` emits CloudWatch Embedded Metric Format records in namespace +`Polis/Math` with P-031's two fixed dimensions `Environment` and `MathEnv`, and +no per-conversation, per-run or per-instance dimension. There is **no AWS client +and no AWS dependency**: a `Sink` receives finished records, and the prototype +ships a JSON-lines sink (`P026_METRICS` = `off` (default), `stderr`, `stdout`, or +a path) plus a null sink. The default is `off` deliberately: a long-running `run` +whose stderr is an undrained pipe blocks once the pipe buffer fills, and per-pass +records fill it far faster than the log lines do, so a deployment picks its sink +explicitly rather than having the coordinator stall on its own telemetry. `polis-coordinator metrics` prints the declared +catalog, which is generated from the emitting code and checked into +`evidence/metrics-catalog.json`. + +**No row claims a P-031 alarm.** Rev7's observability admission is explicit: +none of these series is A01 `PollHealthy` (which needs both poll loops to have +succeeded), A02 `PublishLagSeconds` (initiated-but-unpublished work) or A03 +`ObserverHealthy` (an independent observer). The catalog carries a `p031_status` +block saying so, a Rust test enforces that every row's alarm field is empty, and +`audit_stages.py` re-checks it. Every gauge is scoped to this shard and +allowlist, failures included. + +Per pass: `SourcePassHealthy`, `SourcePassSeconds`, +`SourcePass{Conversations,Probed,Skipped,Reconciled,Published,Deferred}`. Gauges, +at most once per `P026_GAUGE_SECONDS` and computed by one bounded aggregate that +reads no payload column: `OldestReconciliationAgeSeconds` (CO01 scan age), +`ReconciliationBacklogConversations` (CO01 backlog), +`FailureBacklogConversations`, `OldestUnrepairedAgeSeconds` (CO06). Per zid: +`ConversationLatencySeconds`, `SourceReadSeconds`, `ComputeSeconds`, +`PublishSeconds`. Lease outcomes: `LeaseAcquired`, `LeaseUnavailable`, +`LeaseExpired`, `LeaseFenced`. Publication outcomes: `PublishCommitted`, +`PublishConflict`, `PublishRefused`, `PublishRetried`, `PublishUncertain`. Plus +`MetricsDropped`, because a lost record must be visible rather than silent. + +`SourcePassHealthy` says the pass completed, and nothing more: it is page-loop +liveness. A pass in which every conversation failed is still a completed pass; +stuck work is the failure backlog and unrepaired age. + +Source order is the **declared** `polis-order/1` normalization +`(tid,pid,created_ms,semantic_vote,weight_x_32767 NULLS FIRST)`, where +`semantic_vote = raw_vote * storage_agree_value`. It is a contract term, not a +literal in one query: `src/ordering.rs` owns the terms, the coordinator builds +its ORDER BY from them, the same declaration (with its `algorithm_digest` and +this conversation's `equal_time_census`) is the worker manifest's `ordering` +value and is recorded in the published checkpoint, and the worker rejects a +manifest whose declaration does not match its `storage_agree_value`. Ordering on +the raw sign instead would break the polarity property. This is a declared +content normalization, not historical encounter order. Exact duplicate +multiplicity survives. The source fingerprint covers all vote rows, current +comment metadata, participant moderation and that declaration. One +REPEATABLE READ snapshot covers all three source queries. Every conversation is +visited independently of event timestamps; a late commit behind a completed page +is found on the next pass. Source rows are limited to 1,000,000 per table per +conversation; exceeding that limit fails visibly, without acknowledging a prefix. + +`caching_tick` comes from a sequence. It is not commit ordered. The reader always +reserves a metadata sweep page as the unconditional backstop. Its CLI has no +payload cache; selected metadata are cache misses and load full coherent bundles. +Positions survive process restart in `coordinator_cursors`, but remain delivery +hints. Sweep replay recovers a lost consumer response. This does not wire Node's +existing caches to this reader. + +Publication order: parent `conversations FOR KEY SHARE`, lease, ticks, bidtopid, +ptptstats, main. Ownership is checked under the lease lock, and the final +authorization immediately before COMMIT re-reads owner, epoch and the +**remaining** lease under that same row lock, refusing when less than +`P026_COMMIT_MARGIN_SECONDS` is left rather than gambling that the COMMIT round +trip beats the clock. That check does not prove the round trip finishes in time +— nothing in-transaction can — so an uncertain COMMIT is still resolved by +operation, publisher epoch, tick and checkpoint identity, never by a wall-clock deadline. Reclaims increment the epoch; rows are expired, +never deleted/recreated. Default lease duration is 120 seconds. + +## Ownership outcomes + +Refusal is not one state. Each outcome has its own log token and process exit +code, and only one of them is terminal: + +| Outcome | Token | Exit | Meaning | Response | +| --- | --- | --- | --- | --- | +| `LeaseState::Fenced` | `FENCED` | 3 | owner or epoch superseded | publication authority is gone; the daemon exits and this zid is not retried by this owner | +| `LeaseState::Unavailable` | `LEASE-UNAVAILABLE` | 4 | another owner holds an unexpired lease | defer this zid with bounded backoff and keep sweeping the rest | +| `LeaseState::Expired` | `LEASE-EXPIRED` | 5 | our own lease elapsed in DB time | do not publish; reacquire and recompute on a later pass | +| any other failure | — | 1 | not an ownership question | durable backoff, cursor retained | + +Exit codes 4 and 5 are reachable only from `once` and `publish-fixture`, which +are strict one-shot commands: a lease refusal ends that pass with its typed +code. The `run` daemon exits only on `FENCED`. + +A compute that outlasts the lease renews instead of fencing itself. A heartbeat +on its own connection extends `expires_at` every `lease/3`, conditional on +`(owner_id, owner_epoch)` and on the lease still being unexpired in database +time, so it can never resurrect an expired or transferred epoch. A renewal that +fails definitively aborts the operation the worker is running and no publication +follows; an uncertain renewal is not treated as ownership, and after one full +lease without a confirmed renewal the work stops. Ownership is then classified +once more against the lease row before anything is published, so a heartbeat the +database contradicts does not invent a loss, and publication independently +re-checks owner, epoch and expiry inside its own transaction. The heartbeat +cannot renew while the publication transaction holds the lease row `FOR UPDATE`, +so a publication must fit inside one lease window; the pre-commit expiry check is +what enforces that, and it fails closed. + +An unclean death leaves the lease live until it genuinely expires. A restarted +process defers that conversation and keeps working; it never crash-loops, and it +repairs without anyone expiring the dead owner's row by hand. A process that +fails cleanly releases its own epoch immediately (the release is conditional on +`(owner_id, owner_epoch)`, so a transferred row is untouched). That release is +best effort: a failure to reconnect, or a failure before the heartbeat is +started, leaves the lease to expire on its own, so genuine DB-time expiry — not +the release — is what the invariant rests on. + +Per-conversation error attempts are durable and capped at 30 for backoff. SQLSTATE +40001/40P01 gets at most three whole-transaction attempts. Uncertain COMMIT checks +coherent persisted operation/epoch/tick/checkpoint identity before reporting success or failure. + +Integrity digests are separate from harness semantic hashes. JSONB normalizes +numeric spellings (`-0.0` and exponent notation). `storage_digest` expands decimal +number tokens and removes insignificant zeroes, preserving JSON types and all +payload fields. Hashing/encoding occurs before any publication lock. Each results row stores `original_bytes` (BYTEA) and `original_sha256` in the same +publication transaction as `data` (JSONB). These are the admitted worker bytes, +not JSON re-encoded from a parsed value. `load_current` verifies the raw digest, +the checkpoint's raw and storage digests, and the decoded original/JSONB +correspondence before returning a Bundle. The returned originals permit exact +byte replay, including whitespace, negative zero and exponent spellings. Legacy +rows receive nullable columns and fail admission until rebuilt; migration cannot +recover bytes already lost to JSONB. The row-shape golden still compares +`data::text` from the Rust and Python writers. + +**This identity scheme is self-certifying, not tamper-evident.** The digests and +the decoded original/JSONB correspondence catch an *incoherent* generation — a +mutated payload, a deleted companion, a JSONB that has drifted from its original +bytes, another owner's identical content on an uncertain-COMMIT readback. They +catch nothing about a *coherent* forgery: anyone with write access to the four +math tables can rewrite bytes, JSONB, both digests and the checkpoint together, +and this crate will accept the result. Nothing here is signed, and no key is +involved. That is the same trust boundary CO04 v0 already has, and it is stated +here so "original/JSONB mismatch is detectable" is never read as tamper-evidence. + +`publish-fixture` originals are **synthesized** unless the fixture supplies +`payloads.originals` explicitly: the command re-encodes the fixture's parsed +values, announces `originals: synthesized=[…]` on stderr, and for those rows +`validate_originals()` is tautological. Fixture rows are not engine originals and +are not evidence of worker byte custody; `test_s1_identity.py` supplies real +bytes where the byte-replay claim is made. + +## Toolchain + +`rust-toolchain.toml` pins the crate to stable `1.98.1` with `clippy` and +`rustfmt`, so every build here uses one known compiler. Run cargo from inside +`coordinator-rs/` for the pin to apply; `--manifest-path` from the repository +root bypasses it. + +`Cargo.toml` declares `rust-version = "1.88"` as the minimum supported Rust +version. Edition 2024 alone needs 1.85, and the highest MSRV among the locked +dependencies is also 1.85; 1.88 is required because `src/main.rs` uses a +let-chain, stabilized in 1.88 for edition 2024. The MSRV is derived from the +edition, that feature and the lockfile — it has not been exercised by building +on a 1.88 toolchain. Raise it deliberately if newer language features land here. + +## Local build and test + +The sandbox refused writes to `$HOME/.cargo` and `.git/FETCH_HEAD`. Stable rustup +was installed under `/private/tmp/p026-toolchain` with no profile changes. The +Python environment is `/private/tmp/p026-venv`. Dependencies are recorded in +`evidence/python-requirements.txt`; Rust dependencies are in `Cargo.lock`. + +```sh +export CARGO_HOME=/private/tmp/p026-toolchain/cargo +export RUSTUP_HOME=/private/tmp/p026-toolchain/rustup +export PATH="$CARGO_HOME/bin:$PATH" +cargo clippy --manifest-path coordinator-rs/Cargo.toml -- -D warnings +cargo test --manifest-path coordinator-rs/Cargo.toml +cargo build --manifest-path coordinator-rs/Cargo.toml --release +cargo build --manifest-path coordinator-rs/Cargo.toml --features fault-injection --target-dir coordinator-rs/target/fault +COMPOSE_PROJECT_NAME=p026 POLIS_RECOVERY_PG_PORT=55458 RECOVERY_PG_PORT=55458 docker compose -f coordinator-rs/compose.yml up -d --wait +POLIS_TEST_POSTGRES_URL=postgresql://postgres@127.0.0.1:55458/p026 PYTHONPATH=delphi PYTHONDONTWRITEBYTECODE=1 OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 /private/tmp/p026-venv/bin/python -m pytest -o addopts='' --confcutdir=delphi/tests/coordinator delphi/tests/coordinator -q +COMPOSE_PROJECT_NAME=p026 POLIS_RECOVERY_PG_PORT=55458 RECOVERY_PG_PORT=55458 docker compose -f coordinator-rs/compose.yml down -v +``` + +`tools/record_s1.py` reads those cargo transcripts, so every cargo invocation in +the recorded gate must save **both** its output and its own exit status — a +transcript prints one `test result:` line per suite, and a passing suite followed +by a failing one leaves passing lines behind. Run each from inside +`coordinator-rs/` as + +```sh +cargo test --locked > artifacts/s1-cargo-default.log 2>&1; echo $? > artifacts/s1-cargo-default.status +cargo test --locked --features fault-injection > artifacts/s1-cargo-fault.log 2>&1; echo $? > artifacts/s1-cargo-fault.status +cargo clippy --locked --all-targets -- -D warnings > artifacts/s1-clippy-default.log 2>&1; echo $? > artifacts/s1-clippy-default.status +cargo clippy --locked --all-targets --features fault-injection -- -D warnings > artifacts/s1-clippy-fault.log 2>&1; echo $? > artifacts/s1-clippy-fault.status +cargo build --locked --release > artifacts/s1-release.log 2>&1; echo $? > artifacts/s1-release.status +cargo build --locked --features fault-injection --target-dir target/fault > artifacts/s1-fault-build.log 2>&1; echo $? > artifacts/s1-fault-build.status +``` + +A missing `.status` file is malformed evidence, not a pass. The recorder refuses a +non-zero status, any suite whose result is not `ok`, any non-zero `failed` **or** +`ignored` count, a malformed result line and a leading `error:`/`error[` line, and +only then sums the passes. `test_recorder_controls.py` pins those refusals. + +This test-only Compose service binds loopback and uses trust authentication without +credentials. Each test creates a fresh database from a template built by applying +**all repository SQL migrations**. The template removes `math_ticks.caching_tick` +to exercise documented production drift, then applies the prototype migration. +Do not point migration/test commands at an existing service database. + +Commands: `migrate`, `once`, `run`, `read `, `scan `, +`poll `, `reader `, `stages`, `metrics`. `once` completes one full +pass; `run` bounds each cycle by `P026_PAGE_SIZE` (default 16, range 1–1000). +`MATH_ENV` defaults to `rustproto`. `DATABASE_URL`, `P026_PYTHON`, +`STORAGE_AGREE_VALUE` (-1 or +1), `POLL_SHARD_INDEX`, `POLL_SHARD_COUNT`, +`POLL_ALLOWLIST`, `P026_WINDOW` (default 64, positive), `P026_LEASE_SECONDS`, +`P026_POLL_MS`, `P026_CACHE_CAP` (default 16, 0-1024, 0 disables), +`P026_INCREMENTAL` (default 1, 0 disables the probe fast path), +`P026_RECONCILE_SECONDS` (default 3600, positive), `P026_COMMIT_MARGIN_SECONDS` +(default 0.5, less than the lease), `P026_METRICS`, `P026_ENVIRONMENT` (default +`synthetic`) and `P026_GAUGE_SECONDS` are configuration inputs. +`PYTHONPATH` must include this checkout's `delphi` directory. No credentials are +stored in the crate or report. + +## Candidate wire profile + +S1 reserves `polis-input/1` for the complete reviewed contract. This local +profile uses `polis-candidate-input/1`, `polis-candidate-checkpoint/1` and +`polis-candidate-math-output/1`; none is a G01–G16 certificate. + +Initialization also requires the exact admission tuple documented in +`schemas/candidate-admission.schema.json`: candidate schema id, declared engine +version `python-conversation/p026-s1`, SHA256 of the immutable input manifest, +SHA256 of the resolved schedule, and a fresh coordinator operation id. The input +manifest transitively binds vote/moderation files and parent provenance. The +worker validates these before creating Conversation state, returns the tuple at +initialization and in its checkpoint, and Rust compares both against its request. +The engine version is a local adapter revision, not a certified release or a +substitute for the source/dependency hashes recorded in the campaign evidence. + +Admission refusals are typed: `MALFORMED_CANDIDATE`, +`CANDIDATE_SCHEMA_MISMATCH`, `ENGINE_VERSION_MISMATCH`, +`INPUT_DIGEST_MISMATCH`, `SCHEDULE_DIGEST_MISMATCH`, and (at coordinator +readback of the candidate) `OPERATION_ID_MISMATCH`. File checksum refusal remains +`CHECKSUM_MISMATCH`; existing input/sequence/resource errors remain terminal. +Unknown checkpoint or admission fields fail closed. + +The operation id survives whole-transaction retries. Publication writes it and +`publisher_epoch` both in `math_ticks` and the checkpoint; `Bundle` exposes both. +An uncertain COMMIT returns own success only for a coherent current Bundle with +matching operation, publisher epoch, tick, complete checkpoint and all digests. +A different or overwritten publication is `UNCERTAIN_COMMIT_LOST`, never evidence +of our own success. In-place rows provide no historical operation receipt: an +operation overwritten before readback is conservatively lost/unknown. + +Initialization requires descriptors `{path,bytes,sha256}` for an input manifest +and resolved schedule, a `config`, and `required_capabilities`. Dedicated input +and output roots are process arguments. Symlinks/traversal, duplicate keys, invalid +votes (including NULL), foreign identities and checksum errors fail closed. +The control line limit is 64 KiB; each admitted bulk file is bounded at 256 MiB. +The client operation timeout is 120 seconds and failure kills/discards the worker. + +Local manifest keys: `schema:"polis-candidate-input/1"`, `fixture_id`, +`storage_agree_value`, `ordering`, `votes`, `moderation`, `parent`. `ordering` is +either the `polis-order/1` declaration above (live profile) or the pinned name of +a frozen replay order; a live declaration must agree with `storage_agree_value` +and must declare the semantic tie term, or initialization fails. +Vote lines use the contract's eight-field normalized example. Moderation lines +are `{slot,state}`, where state is the existing poller snapshot with four +moderation sets and `lastModTimestamp`. Current poller snapshot semantics leave +the latter null. This profile records real comment `modified` values in source +fingerprints; it does not fabricate historical moderation events. +Schedule: `{schema:"polis-schedule/1",operations:[{op,payload},...]}`. Every +post-initialize operation must match the admitted list exactly. Config keys: +`profile:"candidate-profile"`, `seed:42`, `pca_mode:"powerit"`, +`empty_contract:true`, `init_vector:"engine-default"` (also supports `ones`). +`apply_votes` and `apply_moderation` consume contiguous ranges without compute; +`compute`, `snapshot`, `restore`, `close` are separate operations. Snapshot emits +a manifest last, referring to main, bidtopid, ptptstats and full restore payloads. +The exhaustive B-owned raw schema, source/transform/units manifest, certified +worker provenance and a persisted operation-history restore attestation remain +contract-admission gaps. The local schema must not be advertised as a completed +G01–G16 certificate. + +## Fault seam and evidence + +Build with `fault-injection` into a separate target directory so default builds +cannot replace the running test binary. An attempted release+feature build fails +in `build.rs`. Release binaries reject `P026_FAULT_DIR` before connecting to DB. +The test-enabled binary additionally requires a row for the namespace in +`p026_test_marker` in the target synthetic DB and rejects prod/preprod/dev. + +Write `arm.json` to `P026_FAULT_DIR` with protocol `polis-fault-control/1`, +`run_id`, `operation_id`, `stage`. At the real stage the process atomically writes +`ack.json` carrying PID, context, and `state:"reached-and-blocked"`; it waits for +`release` or an external SIGKILL. Publication contexts contain the actual PG +backend PID. The barrier has a bounded deadline. A log alone is never an ack. + +`audit_stages.py` reports two separate verdicts: `stage_inventory_gate` over the +25 contract-required fault stages, and `full_contract_gate`, which is still +**FAIL**. It names its open conditions explicitly — CO04's `loadBundle`/Bundle +cache-unit rewrite does not exist in the server, application boot/auth/report and +the private served corpus are not executed, `getPca(zid, undefined)` misses a +cold generation zero that the route itself serves, C7's +published-versus-synthesized empty listing must satisfy the existing comment and +clock preservation contract, the incremental probe +is a bounded eligibility hint with no measured service budget, persisted payloads +are revalidated once per ceiling rather than every pass, and this crate +implements none of P-031's A01/A02/A03 — and it exits non-zero while any +remain. + +O8 is the eighth condition and it stays **open**. `evidence/s1-closure.json` +records the S1 work as `state: "PARTIAL"`, and the audit writes +`O8: PARTIAL (S1 identity/custody recorded; G01-G16 open)` into +`condition_states` while leaving the standing "`polis-candidate-input/1` is a +candidate profile, not a G01–G16 certificate" disclaimer in `open_conditions`. +The record names what is still outstanding — the G01–G16 case set with its +negative controls, an immutable manifest, fresh-rebuild versus exact-resume +versus warm-incremental output schedules, and a non-vacuous P-023 compensated +pair with raw C9 validation. `closed_conditions` is empty; nothing here closes. + +The record pins **committed** inputs only — source, migration, schemas, tests — +and the audit re-hashes every one of them and refuses the record if a pin is +stale, missing, or points into gitignored `target/` or `artifacts/`. Binary and +log hashes are still recorded, in a `run_pins` block flagged +`verifiable_on_producing_host_only`, which the audit deliberately does not gate +on: a committed inventory must be reproducible from a clean checkout of the same +commit, not only in the directory that produced it. + +`evidence/` contains sanitized final summaries, fixture digests and the explicit +coverage inventory. Runtime logs and detailed per-test artifacts are retained in +ignored `artifacts/`. A registry with no missing implemented markers does not +claim the absent cache/Node profile stage is covered. diff --git a/coordinator-rs/build.rs b/coordinator-rs/build.rs new file mode 100644 index 0000000000..efa31cc11c --- /dev/null +++ b/coordinator-rs/build.rs @@ -0,0 +1,7 @@ +fn main() { + if std::env::var("PROFILE").as_deref() == Ok("release") + && std::env::var_os("CARGO_FEATURE_FAULT_INJECTION").is_some() + { + panic!("fault-injection is forbidden in release builds"); + } +} diff --git a/coordinator-rs/compose.yml b/coordinator-rs/compose.yml new file mode 100644 index 0000000000..bfe0cbc280 --- /dev/null +++ b/coordinator-rs/compose.yml @@ -0,0 +1,15 @@ +# Synthetic local test database only. Always COMPOSE_PROJECT_NAME=p026. +services: + postgres: + image: postgres:17-alpine + environment: + POSTGRES_USER: postgres + POSTGRES_DB: p026 + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - "127.0.0.1:${POLIS_RECOVERY_PG_PORT:-55458}:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d p026"] + interval: 1s + timeout: 3s + retries: 30 diff --git a/coordinator-rs/evidence/admission.json b/coordinator-rs/evidence/admission.json new file mode 100644 index 0000000000..07c37054e1 --- /dev/null +++ b/coordinator-rs/evidence/admission.json @@ -0,0 +1,67 @@ +{ + "writer_commit": "b3262008f862d155c193b71d7af92b3d8e3d37f8", + "recovery_remote_tip": "a4f3b4d82bad53b5039979db9a3b29ed276a69b5", + "recovery_local_oracle": "aaaf7ca5c93f9a758b28e7a361c3b9544e24305a", + "checkout_base": "451ab8265a244ba5cfeb1888a375ccba911682e7", + "input_fixtures": { + "vw": { + "sha256": "622b8c164150dc509f7387d81d2e2d616d37211c3bb256458064972ad805b56a", + "vote_rows": 4683, + "source_format": "public export CSV", + "normalization": "replay.real_data.load_export_votes; storage vote=-semantic sign; timestamps in milliseconds", + "synthetic_zid": 1 + }, + "biodiversity": { + "sha256": "9b6c1847bcd2e93c2cc349e79676f8be3ffe49a7e5f79f0d2e19412f0be29f94", + "vote_rows": 29802, + "source_format": "public export CSV", + "normalization": "replay.real_data.load_export_votes; storage vote=-semantic sign; timestamps in milliseconds", + "synthetic_zid": 1 + } + }, + "files": { + "coordinator-rs/src/config.rs": "6e8ff6c9165e83109ec4cdba22113625b4cebda568261146c74bc50e4f3dd46f", + "coordinator-rs/src/lib.rs": "c0b414f0fa1982004862e25958e7e650174f613c99977a3caa63b7c74e0c0649", + "coordinator-rs/src/store.rs": "83dbd47fa8019fdb9edf053d64d7bc3cc0c88ceeb8089097c0afffcc9f8a3ef7", + "coordinator-rs/src/coordinator.rs": "3167c58df6faa62e45bc0fdbecf87c2d5cd70975815f05620b9ae8ef4354a045", + "coordinator-rs/src/fault.rs": "c39d73439b22cb343d18f87296c3f78e809e53c0b637fab5ed3c5cb084d70dc7", + "coordinator-rs/src/main.rs": "9bdd1f79efa671f79d25b98f0530aab569d62803d6d49a1e38cbd84638bfefa7", + "coordinator-rs/src/wire.rs": "5065d751edbd17987222b3cf18c262a669ec7df96afa25979c88422288972013", + "coordinator-rs/src/reader.rs": "07f70f308dc018fdd9b95c6deda5a0c38053e049acb391e774d4d70401dd412c", + "coordinator-rs/src/engine.rs": "73ce227f5ce5f2e2c111ec5f757f9c1c73ca04fbbf6df4ff43320269e7ceb1d3", + "coordinator-rs/Cargo.toml": "87d4c0a11cf171daf946519514d07512c7da9b7ccc9a428a1ee435881b7f8554", + "coordinator-rs/Cargo.lock": "308d38666623b7bc9a0f5f09ae2084dfeef667c977a9ee241df3003d6e1306bf", + "coordinator-rs/migration.sql": "da000ce00b7cb761e758283f555b9cd66eef096f73bba35eb9e74288c0e69e07", + "delphi/polismath/engine_adapter.py": "b4198c15fdc3c85344e5b0785289588227c1706167d96110c491249896df25c3", + "coordinator-rs/build.rs": "f717ec6adfab1adb10fdae12db4b9d19d3de06265603ef5112d1336e243598fd", + "delphi/tests/coordinator/audit_stages.py": "55d7ef73dff047119249f81538f6f72c18297b4de7f4f99b0bd5705a7a693ca5", + "delphi/tests/coordinator/conftest.py": "b04a75a8f01adbc74d4a9e70d3e79ea7fc5d44a32c2e0a7a931235605b854d84", + "delphi/tests/coordinator/reference_child.py": "add6c53bbe9bcb4b384bdb1a80b31ca51dcce0eabe1ead4d31d2b3376a25f6f5", + "delphi/tests/coordinator/test_adapter.py": "df306b2813033470d98446ab8be1dcb66c174488d6f6b4d3dd8bfa6f0fdb3771", + "delphi/tests/coordinator/test_equivalence.py": "e4b43ca75474fc05bca1098b5483d94de3e7c05ecc51fb9cdf837e4f9a08bfa6", + "delphi/tests/coordinator/test_recovery.py": "00a7a5861eabed883100daf54abce8a4c2f2ee0b8928d1f78e85aa151551d7af", + "delphi/tests/coordinator/test_reference_controls.py": "3e4ea890b96f7c604ea53f72a4227bbe1a7929dddd267e10fa0e173cfe4a52eb", + "delphi/tests/coordinator/test_store.py": "ffd1a2ad2ffab5d250ecfcf0a3d44e2da93e8931c9b8176900e13f41714d8310", + "coordinator-rs/tests/validation.rs": "7d90220e5033e207426a82096ae673ba1e35a929c08792569de9531fa9d69ad3" + }, + "binaries": { + "coordinator-rs/target/release/polis-coordinator": "03074e065822ccb34e7fad0ccf401c587274deaaf47e6ce8499a499809697ab3", + "coordinator-rs/target/fault/debug/polis-coordinator": "6d9e442dd7dda7e5dae46ccf5e09f0cdff3fef993d79efee1e5a3d1a39de45d2" + }, + "runtime": { + "rustc": "1.98.1", + "python": "3.12.11", + "postgres_major": 17, + "compose_project": "p026", + "postgres_port": 55458, + "engine_seed": 42, + "engine_preset": "engine-default", + "storage_agree_value": -1, + "test_page_size": 2, + "test_poll_window": 1, + "math_namespaces": [ + "python", + "rustproto" + ] + } +} diff --git a/coordinator-rs/evidence/d4-generation-zero.json b/coordinator-rs/evidence/d4-generation-zero.json new file mode 100644 index 0000000000..c1e7326789 --- /dev/null +++ b/coordinator-rs/evidence/d4-generation-zero.json @@ -0,0 +1,30 @@ +{ + "committed_math_tick": 0, + "note": "the route requests latest with math_tick=-1 and serves generation zero; getPca(zid, undefined) is a different branch and is cache-dependent. That helper asymmetry is a reader defect for the server owner, characterised here, not fixed here.", + "profile": "real handle_GET_math_pca2 on a real Express app over loopback HTTP, real reader/cache/PostgreSQL; only the parameter middleware is synthetic", + "route": { + "coldUndefinedPresent": false, + "conditional": { + "bodyTick": null, + "contentEncoding": "identity", + "n": null, + "status": 304 + }, + "pgTickType": "string", + "subset": { + "bodyTick": 0, + "contentEncoding": "identity", + "etag": "\"0\"", + "n": 6, + "status": 200 + }, + "warmUndefinedPresent": true, + "whole": { + "bodyTick": 0, + "contentEncoding": "gzip", + "etag": "\"0\"", + "n": 6, + "status": 200 + } + } +} \ No newline at end of file diff --git a/coordinator-rs/evidence/d4-node-reader-empty.json b/coordinator-rs/evidence/d4-node-reader-empty.json new file mode 100644 index 0000000000..80b271b03a --- /dev/null +++ b/coordinator-rs/evidence/d4-node-reader-empty.json @@ -0,0 +1,104 @@ +{ + "differences": { + "asJSON_bytes": [ + 753, + 455 + ], + "asJSON_sha256": [ + "fa78e3c4a1cbacb495960da1fcb9b1a834ffc0d825361b968e9e2eadb2670050", + "dee184361aca0f04b5a3e19c5c08f10874b85cba832d4d5bd71d3b22b316de2e" + ], + "bid_to_pid": [ + [], + null + ], + "gzip_bytes": [ + 343, + 266 + ], + "gzip_sha256": [ + "666e7e9f166083dfc4cf0489725d1be564b371049fc0a3779bdcf67db8cb76f9", + "0a3046153f19f3c1bdf154d66e523e5d0ce0352c327984e0fb133b1f0bb2ca39" + ], + "keys_projection_sha256": [ + "5e21b0f6fec91b0e609a968a8c308d8c57e58062a487dbff98628b8206acfac7", + "bdfafdc98f5d9e7439255a1073827ca5509cc40621dbb02bd1e9e8f5d7668a31" + ], + "mapping_is_error": [ + false, + true + ], + "mapping_sha256": [ + "a5558a1314571ed26f6afdea59f81a24ab0cc942ac84485892f8a36f7ac16315", + null + ], + "tids": [ + [], + [ + 0, + 1, + 2, + 3 + ] + ] + }, + "note": "the synthesized empty presentation stamps lastVoteTimestamp with Date.now(), so it is not reproducible across servers or requests; the published empty generation is", + "profile": "zero votes, four approved comments; rustproto has a published empty generation, python has none and is synthesized by the server", + "reference_published": false, + "served": { + "python": { + "asJSON_bytes": 455, + "asJSON_sha256": "dee184361aca0f04b5a3e19c5c08f10874b85cba832d4d5bd71d3b22b316de2e", + "bid_to_pid": null, + "consensus_shape": { + "agree": 0, + "disagree": 0 + }, + "etag_math_tick": 0, + "gzip_bytes": 266, + "gzip_sha256": "0a3046153f19f3c1bdf154d66e523e5d0ce0352c327984e0fb133b1f0bb2ca39", + "keys_projection_sha256": "bdfafdc98f5d9e7439255a1073827ca5509cc40621dbb02bd1e9e8f5d7668a31", + "last_vote_timestamp": 1788891903170, + "mapping_is_error": true, + "mapping_math_tick": null, + "mapping_sha256": null, + "math_env": "python", + "n": 0, + "pids_for_gid": { + "0": [] + }, + "present": true, + "repness_keys": [], + "tids": [ + 0, + 1, + 2, + 3 + ] + }, + "rustproto": { + "asJSON_bytes": 753, + "asJSON_sha256": "fa78e3c4a1cbacb495960da1fcb9b1a834ffc0d825361b968e9e2eadb2670050", + "bid_to_pid": [], + "consensus_shape": { + "agree": 0, + "disagree": 0 + }, + "etag_math_tick": 1, + "gzip_bytes": 343, + "gzip_sha256": "666e7e9f166083dfc4cf0489725d1be564b371049fc0a3779bdcf67db8cb76f9", + "keys_projection_sha256": "5e21b0f6fec91b0e609a968a8c308d8c57e58062a487dbff98628b8206acfac7", + "last_vote_timestamp": 0, + "mapping_is_error": false, + "mapping_sha256": "a5558a1314571ed26f6afdea59f81a24ab0cc942ac84485892f8a36f7ac16315", + "math_env": "rustproto", + "n": 0, + "pids_for_gid": { + "0": [] + }, + "present": true, + "repness_keys": [], + "tids": [] + } + } +} \ No newline at end of file diff --git a/coordinator-rs/evidence/d4-node-reader.json b/coordinator-rs/evidence/d4-node-reader.json new file mode 100644 index 0000000000..f85d208a69 --- /dev/null +++ b/coordinator-rs/evidence/d4-node-reader.json @@ -0,0 +1,129 @@ +{ + "compared_fields": [ + "asJSON_sha256", + "asJSON_bytes", + "gzip_sha256", + "gzip_bytes", + "keys_projection_sha256", + "tids", + "n", + "repness_keys", + "consensus_shape", + "mapping_sha256", + "bid_to_pid", + "pids_for_gid", + "mapping_is_error" + ], + "differences": {}, + "math_tick": 1, + "profile": "real server/src/utils/pca.ts + participants.ts, in-process, one Node process", + "served": { + "python": { + "asJSON_bytes": 6467, + "asJSON_sha256": "a6f97eefa01d1163618efa72aef0d84daa82816185952dca43f2d0c5f04e4e6f", + "bid_to_pid": [ + [ + 0 + ], + [ + 1, + 4 + ], + [ + 2, + 5 + ], + [ + 3 + ] + ], + "consensus_shape": { + "agree": 0, + "disagree": 0 + }, + "etag_math_tick": 1, + "gzip_bytes": 1736, + "gzip_sha256": "bf1011c0dcffd052f512135046e4c38f807c2158c82faaea6f45cc912110f528", + "keys_projection_sha256": "08743f5ed0bac5e02c7a38535745cbc2c5d22f0b8c52d601dda7c65b4c9dd29a", + "mapping_is_error": false, + "mapping_sha256": "9a6fbcea6bfcd81da7c209e594312ba9860280003a57dd122943ca009253dedd", + "n": 6, + "pids_for_gid": { + "0": [ + 0, + 2, + 3, + 5 + ], + "1": [ + 1, + 4 + ] + }, + "repness_keys": [ + "0", + "1" + ], + "tids": [ + 0, + 1, + 2, + 3 + ] + }, + "rustproto": { + "asJSON_bytes": 6467, + "asJSON_sha256": "a6f97eefa01d1163618efa72aef0d84daa82816185952dca43f2d0c5f04e4e6f", + "bid_to_pid": [ + [ + 0 + ], + [ + 1, + 4 + ], + [ + 2, + 5 + ], + [ + 3 + ] + ], + "consensus_shape": { + "agree": 0, + "disagree": 0 + }, + "etag_math_tick": 1, + "gzip_bytes": 1736, + "gzip_sha256": "bf1011c0dcffd052f512135046e4c38f807c2158c82faaea6f45cc912110f528", + "keys_projection_sha256": "08743f5ed0bac5e02c7a38535745cbc2c5d22f0b8c52d601dda7c65b4c9dd29a", + "mapping_is_error": false, + "mapping_sha256": "9a6fbcea6bfcd81da7c209e594312ba9860280003a57dd122943ca009253dedd", + "n": 6, + "pids_for_gid": { + "0": [ + 0, + 2, + 3, + 5 + ], + "1": [ + 1, + 4 + ] + }, + "repness_keys": [ + "0", + "1" + ], + "tids": [ + 0, + 1, + 2, + 3 + ] + } + }, + "zid": 1 +} \ No newline at end of file diff --git a/coordinator-rs/evidence/metrics-catalog.json b/coordinator-rs/evidence/metrics-catalog.json new file mode 100644 index 0000000000..d737298e60 --- /dev/null +++ b/coordinator-rs/evidence/metrics-catalog.json @@ -0,0 +1 @@ +{"dimensions":["Environment","MathEnv"],"metrics":[{"meaning":"1 only when a whole source pass completed; 0 on a failed pass, missing when the process is gone. This is page-loop liveness. It is NOT P-031 A01 PollHealthy, which requires both the vote and the moderation poll to have succeeded recently: a pass in which every conversation failed still completes","name":"SourcePassHealthy","p031_alarm":"","statistic":"Minimum","unit":"Count"},{"meaning":"wall time of one bounded source pass","name":"SourcePassSeconds","p031_alarm":"","statistic":"Maximum","unit":"Seconds"},{"meaning":"conversations visited by the pass, before shard/backoff filtering","name":"SourcePassConversations","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"conversations whose cheap change probe was evaluated","name":"SourcePassProbed","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"conversations the probe found unchanged, so no full source read was taken","name":"SourcePassSkipped","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"conversations that took the authoritative full snapshot this pass","name":"SourcePassReconciled","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"conversations that committed a new generation this pass","name":"SourcePassPublished","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"conversations deferred this pass with durable bounded backoff","name":"SourcePassDeferred","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"CO01 scan age: age of the oldest authoritative full source snapshot for this shard's candidates, measured from before the source read. The incremental fast path is only sound while this is bounded. It is NOT P-031 A02 PublishLagSeconds: a quiet conversation's source age grows toward the ceiling with no pending math at all","name":"OldestReconciliationAgeSeconds","p031_alarm":"","statistic":"Maximum","unit":"Seconds"},{"meaning":"CO01 backlog: candidate conversations never reconciled, or overdue for reconciliation","name":"ReconciliationBacklogConversations","p031_alarm":"","statistic":"Maximum","unit":"Count"},{"meaning":"CO01 failures: candidate conversations currently in durable backoff; a conversation this shard/allowlist would never attempt is not counted","name":"FailureBacklogConversations","p031_alarm":"","statistic":"Maximum","unit":"Count"},{"meaning":"CO06 oldest unrepaired age: how long the oldest still-failing candidate conversation has been failing. Scoped to this shard and allowlist, like every other gauge here. Not P-031 A02","name":"OldestUnrepairedAgeSeconds","p031_alarm":"","statistic":"Maximum","unit":"Seconds"},{"meaning":"per-zid probe -> source -> compute -> publish wall time","name":"ConversationLatencySeconds","p031_alarm":"","statistic":"Maximum","unit":"Seconds"},{"meaning":"per-zid authoritative snapshot read time","name":"SourceReadSeconds","p031_alarm":"","statistic":"Maximum","unit":"Seconds"},{"meaning":"per-zid worker compute time, worker process lifetime included","name":"ComputeSeconds","p031_alarm":"","statistic":"Maximum","unit":"Seconds"},{"meaning":"per-zid publication transaction time, retries included","name":"PublishSeconds","p031_alarm":"","statistic":"Maximum","unit":"Seconds"},{"meaning":"CO03 lease outcome: this owner took or renewed ownership at an epoch","name":"LeaseAcquired","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"CO03 lease outcome: another owner holds an unexpired lease","name":"LeaseUnavailable","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"CO03 lease outcome: this owner's own lease elapsed in database time","name":"LeaseExpired","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"CO03 lease outcome: owner or epoch superseded; terminal for this owner","name":"LeaseFenced","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"CO04 publication outcome: one generation committed","name":"PublishCommitted","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"CO04 publication outcome: stale expected tick, nothing overwritten","name":"PublishConflict","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"CO04 publication outcome: ownership refused inside the transaction","name":"PublishRefused","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"CO04 publication outcome: whole-transaction retry after 40001/40P01","name":"PublishRetried","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"CO04 publication outcome: lost COMMIT response resolved by checkpoint identity","name":"PublishUncertain","p031_alarm":"","statistic":"Sum","unit":"Count"},{"meaning":"records this process failed to write; a nonzero value means the other series are incomplete","name":"MetricsDropped","p031_alarm":"","statistic":"Sum","unit":"Count"}],"namespace":"Polis/Math","p031_status":{"coverage_claimed":[],"not_implemented":["A01 PollHealthy","A02 PublishLagSeconds","A03 ObserverHealthy"],"note":"These are local diagnostics with no deployed publisher and no delivery proof. A01 needs both poll loops to have succeeded, A02 needs initiated-but-unpublished work, and A03 needs an independent observer; none of the series below is any of those. Scope: every gauge is scoped to this shard and allowlist."},"schema":"polis-metrics/1","transport":"CloudWatch Embedded Metric Format records on a JSON-lines sink; no AWS client in this crate"} diff --git a/coordinator-rs/evidence/polarity-biodiversity.json b/coordinator-rs/evidence/polarity-biodiversity.json new file mode 100644 index 0000000000..d135320f4a --- /dev/null +++ b/coordinator-rs/evidence/polarity-biodiversity.json @@ -0,0 +1,7 @@ +{ + "fixture": "biodiversity", + "a": "a1ce75416a464c4562f7fc69c929ed9f018fedd207d77d8dab355c378bedec94", + "b": "a1ce75416a464c4562f7fc69c929ed9f018fedd207d77d8dab355c378bedec94", + "negative": "fd0b3ef30a208f561acf3eb066e2c2471e1296c1b66f44aeb52d7b78a9f22284", + "deltas": [] +} \ No newline at end of file diff --git a/coordinator-rs/evidence/polarity-rebuild-schedule.json b/coordinator-rs/evidence/polarity-rebuild-schedule.json new file mode 100644 index 0000000000..0f75bd301d --- /dev/null +++ b/coordinator-rs/evidence/polarity-rebuild-schedule.json @@ -0,0 +1,20 @@ +[ + { + "step": 0, + "positive": "30a1d16e462192a65666b786beee64bf2997f9928f9b28a1b38dbf310dacc74d", + "paired": "30a1d16e462192a65666b786beee64bf2997f9928f9b28a1b38dbf310dacc74d", + "negative": "01a9fdf049f73ce57d8ba66380b80c622e09d2ebe8a896aa7dbd09beca03baec" + }, + { + "step": 1, + "positive": "f71a5ad62ac47f2133d4158ba31d920d1ffae102d0684820ed3c81bbf941dea8", + "paired": "f71a5ad62ac47f2133d4158ba31d920d1ffae102d0684820ed3c81bbf941dea8", + "negative": "c75d7daa88d10dc30720513e7b3573c308d4b0c20c253d9577b0df638a4ecf11" + }, + { + "step": 2, + "positive": "efe1d6ef80fcd3c076c08ce95cf2cc4dcbc6f9003e8b55a6efe50d6721589e4d", + "paired": "efe1d6ef80fcd3c076c08ce95cf2cc4dcbc6f9003e8b55a6efe50d6721589e4d", + "negative": "07290dd4ad42370888a46f21c7cc247b4daf12a50d3f49812ddb46e891e2c418" + } +] \ No newline at end of file diff --git a/coordinator-rs/evidence/polarity-synthetic.json b/coordinator-rs/evidence/polarity-synthetic.json new file mode 100644 index 0000000000..26159ee025 --- /dev/null +++ b/coordinator-rs/evidence/polarity-synthetic.json @@ -0,0 +1,7 @@ +{ + "fixture": "synthetic", + "a": "2fb950bae8230a3e781feb724ffa27569eacb80ec3c786113750deed1cbd642e", + "b": "2fb950bae8230a3e781feb724ffa27569eacb80ec3c786113750deed1cbd642e", + "negative": "959731736e87e7e23c20261dda26937c6d3474b1e4c8d5267ee11c20e3558202", + "deltas": [] +} \ No newline at end of file diff --git a/coordinator-rs/evidence/polarity-vw.json b/coordinator-rs/evidence/polarity-vw.json new file mode 100644 index 0000000000..8914f416a4 --- /dev/null +++ b/coordinator-rs/evidence/polarity-vw.json @@ -0,0 +1,7 @@ +{ + "fixture": "vw", + "a": "d401fc7095d77708eb56aa45aa49f5dc819425a5c03e85bac8894f43cbee8b5c", + "b": "d401fc7095d77708eb56aa45aa49f5dc819425a5c03e85bac8894f43cbee8b5c", + "negative": "a889972e8feda7f82144333115e3ce28657c243aa348ef0365026b463afdb9b4", + "deltas": [] +} \ No newline at end of file diff --git a/coordinator-rs/evidence/python-requirements.txt b/coordinator-rs/evidence/python-requirements.txt new file mode 100644 index 0000000000..95cb6e2d6c --- /dev/null +++ b/coordinator-rs/evidence/python-requirements.txt @@ -0,0 +1,35 @@ +boto3==1.43.89 +botocore==1.43.89 +cloudpickle==3.1.2 +contourpy==1.3.3 +cycler==0.12.1 +fonttools==4.64.0 +iniconfig==2.3.0 +jmespath==1.1.0 +joblib==1.6.0 +kiwisolver==1.5.1 +matplotlib==3.11.1 +narwhals==2.25.0 +natsort==8.4.0 +numpy==1.26.4 +packaging==26.3 +pandas==2.3.3 +pillow==12.3.0 +pluggy==1.6.0 +psycopg2-binary==2.9.12 +pygments==2.21.0 +pyparsing==3.3.2 +pytest==9.1.1 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.3 +pytz==2026.3.post1 +pyyaml==6.0.3 +s3transfer==0.19.2 +scikit-learn==1.9.0 +scipy==1.17.1 +six==1.17.0 +sqlalchemy==2.0.52 +threadpoolctl==3.6.0 +typing-extensions==4.16.0 +tzdata==2026.3 +urllib3==2.7.0 diff --git a/coordinator-rs/evidence/s1-audit-controls.json b/coordinator-rs/evidence/s1-audit-controls.json new file mode 100644 index 0000000000..6a985b2fea --- /dev/null +++ b/coordinator-rs/evidence/s1-audit-controls.json @@ -0,0 +1,146 @@ +{ + "negative_controls": [ + { + "control": "stale-source-pin", + "refused": true, + "O8": "OPEN", + "full_contract_gate": "FAIL", + "open_conditions": 8, + "reason": "coordinator-rs/src/store.rs" + }, + { + "control": "missing-source-pin", + "refused": true, + "O8": "OPEN", + "full_contract_gate": "FAIL", + "open_conditions": 8, + "reason": "missing S1 source pins" + }, + { + "control": "gitignored-build-output-pinned-as-source", + "refused": true, + "O8": "OPEN", + "full_contract_gate": "FAIL", + "open_conditions": 8, + "reason": "gitignored build output pinned as a source: coordinator-rs/target/fault/debug/polis-coordinator" + }, + { + "control": "record-claims-CLOSED", + "refused": true, + "O8": "OPEN", + "full_contract_gate": "FAIL", + "open_conditions": 8, + "reason": "S1 partial record refused:" + }, + { + "control": "record-names-no-remaining-obligations", + "refused": true, + "O8": "OPEN", + "full_contract_gate": "FAIL", + "open_conditions": 8, + "reason": "a PARTIAL record must name what is still open" + } + ], + "restored_O8": "PARTIAL (S1 identity/custody recorded; G01-G16 open)", + "restored_open_conditions": 8, + "restored_refusal": false, + "clean_copy_reverification": { + "method": "rsync of the committed tree excluding coordinator-rs/target/, coordinator-rs/artifacts/, node_modules and .git; fault binary rebuilt from source; only stage-*.json copied in; no s1-*.log, no s1-pytest.xml, no release binary", + "rebuilt_binary_hash_differs_from_run_pins": true, + "O8": "PARTIAL (S1 identity/custody recorded; G01-G16 open)", + "open_conditions": 8, + "full_contract_gate": "FAIL", + "inventory_summary_identical_to_producing_host": true + }, + "audit_inventory_summary_agree": true, + "plan_checker_controls": { + "checker": "cost-reduction/scripts/p026-step4-plan-check.py (out of tree)", + "invariant": "len(open_conditions) + len(closed_conditions) == 8 declared conditions; a partial_conditions entry discharges nothing, so its condition must still be listed open", + "run_on_every_invocation": true, + "controls": [ + { + "control": "condition-closed-and-recorded", + "expectation": "accepted", + "result": "accepted", + "reason": null + }, + { + "control": "condition-dropped-entirely", + "expectation": "refused", + "result": "refused", + "reason": "control: 7 open + 0 closed != 8 declared (partial: 0); a condition was dropped or double-counted" + }, + { + "control": "partial-removed-from-open", + "expectation": "refused", + "result": "refused", + "reason": "control: PARTIAL record discharges nothing, so its condition must stay in open_conditions: O8 remains OPEN and is at most PARTIAL: polis-candidate-input/1 is cla..." + } + ], + "latent_defect_regression": { + "shape": "c50f76216: 7 open + 1 closed, the audit having popped a discharged condition", + "old_check": "compared the 8-entry declaration to open_conditions directly", + "old_result": "FAIL (source=8, evidence=7) - permanent once anything closes", + "new_result": "PASS" + } + }, + "recorder_cargo_controls": { + "parser": "coordinator-rs/tools/record_s1.py rust_test_counts()", + "rule": "captured per-invocation cargo exit status must be 0; every suite in the transcript must be `ok` with 0 failed and 0 ignored; a malformed result line or a leading error: line is refused; only then are the passes summed. No fixed total is reinstated.", + "pinned_by": "delphi/tests/coordinator/test_recorder_controls.py (8 cases, in the Python set)", + "controls": [ + { + "control": "multi-suite-success", + "expectation": "accepted", + "result": "accepted", + "counts": { + "default": 31, + "fault": 31 + }, + "reason": null + }, + { + "control": "pass-then-fail (the reviewed defect)", + "expectation": "refused", + "result": "refused", + "counts": null, + "reason": "failed cargo suite in s1-cargo-default.log: FAILED" + }, + { + "control": "ignored-test-not-a-pass", + "expectation": "refused", + "result": "refused", + "counts": null, + "reason": "1 ignored cargo tests in s1-cargo-default.log" + }, + { + "control": "nonzero-cargo-exit-status", + "expectation": "refused", + "result": "refused", + "counts": null, + "reason": "s1-cargo-default exited 101" + }, + { + "control": "missing-cargo-exit-status", + "expectation": "refused", + "result": "refused", + "counts": null, + "reason": "no captured exit status for s1-cargo-fault" + }, + { + "control": "malformed-result-line", + "expectation": "refused", + "result": "refused", + "counts": null, + "reason": "malformed cargo test result line in s1-cargo-default.log" + }, + { + "control": "no-result-line", + "expectation": "refused", + "result": "refused", + "counts": null, + "reason": "no cargo test result line in s1-cargo-default.log" + } + ] + } +} diff --git a/coordinator-rs/evidence/s1-closure.json b/coordinator-rs/evidence/s1-closure.json new file mode 100644 index 0000000000..d52adb1ffb --- /dev/null +++ b/coordinator-rs/evidence/s1-closure.json @@ -0,0 +1,160 @@ +{ + "id": "O8", + "state": "PARTIAL", + "scope": "P-026 step-4 S1 candidate identity and original-byte custody", + "g01_g16_certified": false, + "rust_equals_clojure_claimed": false, + "candidate_schema": "polis-candidate-input/1", + "engine_version": "python-conversation/p026-s1", + "recorded": [ + "distinct candidate schema id and strict adapter admission on both sides", + "original worker bytes and their digests stored atomically with the JSONB, with decoded original/JSONB correspondence checked on read", + "worker/engine identity re-verified at initialization and at snapshot", + "operation id and publisher epoch bound into the uncertain-COMMIT readback" + ], + "remaining_obligations": [ + "all applicable G01-G16 cases and their required negative controls", + "an immutable manifest for the profile", + "fresh rebuild, exact resume and warm incremental output schedules distinguished", + "a non-vacuous P-023 compensated pair and raw C9 validation" + ], + "fixture_originals": "publish-fixture synthesizes original bytes by re-encoding the fixture's parsed values when no explicit originals are supplied; those rows are labelled synthesized on stderr and validate_originals() is tautological for them. They are not engine originals.", + "integrity_model": "self-certifying, not tamper-evident: a writer with access to the four math tables can rewrite bytes, JSONB, both digests and the checkpoint coherently and is not detected. Digests bind an honest writer's own output; they are not an authentication scheme.", + "pins_are_committed_sources": true, + "sha256": { + "coordinator-rs/Cargo.lock": "308d38666623b7bc9a0f5f09ae2084dfeef667c977a9ee241df3003d6e1306bf", + "coordinator-rs/Cargo.toml": "b368a7e118a13200d5fc95e86a0324a3a3d5f0f233b1b01688b0f62d3fd30187", + "coordinator-rs/evidence/python-requirements.txt": "6bd7073c45722296d4ababbf7c635c3346067b4ef30f9893cfc7bed55a0166e5", + "coordinator-rs/migration.sql": "e53f42cc858d064b8280f6f7dcad63dd272de8260633581520e488a6b47b0a55", + "coordinator-rs/rust-toolchain.toml": "0d8f9ab90bbeff10ce949b8ecaaf947712feca7320411a53f44d400e4ecb4860", + "coordinator-rs/schemas/candidate-admission.schema.json": "fa641bac10932404ec40ea3441585f5f9d72e8be6facc61b650ee5f2d82fcb46", + "coordinator-rs/src/cache.rs": "3aac2ad841b28e98a8adfb01975cdfb2ab39beff87d189e549cc534d3300ccf5", + "coordinator-rs/src/config.rs": "a0e15de96c79a03d4659e763ea2275d4f1371d218cd75792e910c6b1254506d9", + "coordinator-rs/src/coordinator.rs": "e2604e9eb9e9e41f65647a59bb8ed91e28ca4002952dff3acdf6a22c29ba7e53", + "coordinator-rs/src/engine.rs": "e2467cb0c68986cd76782245be9576bdb97026d27d5d6f8b128a9ff93079f2b2", + "coordinator-rs/src/fault.rs": "e90c29508b3785fc9208882fcb69f7d3dad16d761be3118d65b24ab874faa059", + "coordinator-rs/src/lease.rs": "e282ff4105abbef0b3aad3bec52652a5174e0383100c636eb7815ed9bcf6d784", + "coordinator-rs/src/lib.rs": "7037d045d4badf7a3a1266f2322e2d4b701040e5042637e0cbf6b2980a50d467", + "coordinator-rs/src/main.rs": "bb9fb3ad9bbe0f046cd208e7d8a658369036a6d9b7441bb83e7cae27c739a2fb", + "coordinator-rs/src/metrics.rs": "97c37d9c77962d2f66d46cc22c43c623d974b4b39c99eec8e1e7748f0fef6eef", + "coordinator-rs/src/ordering.rs": "16e4e867a82917e15857855e80ef59b992f280b237e46432279092c51a61bfd7", + "coordinator-rs/src/probe.rs": "f6a8310647f798db60c8638deb690a8f8a71e0b0777adc7162715f60ef63fdfe", + "coordinator-rs/src/reader.rs": "07f70f308dc018fdd9b95c6deda5a0c38053e049acb391e774d4d70401dd412c", + "coordinator-rs/src/store.rs": "ab6f2c2fb0b9c902e3e63abec9bf80f636f8c58b9a87b67b05eb090da380364f", + "coordinator-rs/src/wire.rs": "5065d751edbd17987222b3cf18c262a669ec7df96afa25979c88422288972013", + "coordinator-rs/tests/s1_admission.rs": "729a351dcaa6aab1a57895073d6c5ccf03f75a34852be566097f1c27e9c4ae18", + "coordinator-rs/tests/validation.rs": "d2a6230752d78dbc57c57d2dca1c88ff887ff1200303b02bfc0f87e4c03fe7d5", + "coordinator-rs/tools/record_s1.py": "61a63ad00a4fd4f1a358453fbf73fb3f3a119c68bdf07e789773a69869f26f26", + "delphi/polismath/__init__.py": "77261d0fc27172aa576a06d9947330f2d5b1bcd8d14721df98eac7c5f305f16b", + "delphi/polismath/benchmarks/__init__.py": "ac90a49e001cba824dc10b56e6f67d671087eaf458d6ff5616c017dedd4b7a98", + "delphi/polismath/benchmarks/bench_pca.py": "98f0e73cbf06b3859ec530f0a94427794748d8d27d66e2ecfe10801d5293d340", + "delphi/polismath/benchmarks/bench_repness.py": "27582505d8140e8efd0e7c3edf035baad48ecc63cda9e4c668dfbdfeff8eef6f", + "delphi/polismath/benchmarks/bench_update_votes.py": "614a49137e2e49f4b21b523d3ceee14132a0fa3cad24ac5acbda7a3f152eb23e", + "delphi/polismath/benchmarks/benchmark_utils.py": "73455f256a3f48a8be736faf8c704e5bbe3bc327f92463595210f09d11ccc961", + "delphi/polismath/components/__init__.py": "19f991f755b893897858c2f3cf0d4be8f56db72fd3c4aa3c24221c4357a52bea", + "delphi/polismath/components/config.py": "2fbbf0d3775a0227f3a31c14f59bb9257651c6e6108ca4b595cd260560826fff", + "delphi/polismath/conversation/__init__.py": "af4a23759af7e52732e2da8e4e0ce3371c8657d45bd70ade366693e64a4115c7", + "delphi/polismath/conversation/conversation.py": "f4625456f42d7dbbdcdebd7fc8acf8dcb59377c77cd6030bbfaca4d924db7f22", + "delphi/polismath/conversation/manager.py": "5ab0d705c141f0c0e96472750e0fbe92bdb86ab65049889041cd198801d80059", + "delphi/polismath/database/__init__.py": "26a51d925716a172271fa53538b0209fe4ab96fa690a9f23ac4e46664502dd9a", + "delphi/polismath/database/dynamodb.py": "8d79e40b81d7986c849cacca93481eafe35e6432174c6f31b57c1d107e69db3b", + "delphi/polismath/database/postgres.py": "371b8177fbe0b49f8cbbc5ba37911cd98bd69178630dd5611630d9849111efec", + "delphi/polismath/engine_adapter.py": "6ade0aac5cc3da947b3c3377473a96b91cb67945cf4eedfba3f1fbf037b74d63", + "delphi/polismath/pca_kmeans_rep/__init__.py": "780ed50145152784fdd316c58fe59e3535607fedcc78414d6d0e37b6b3f8d3b1", + "delphi/polismath/pca_kmeans_rep/clusters.py": "8d1d1278fdc6788443772a8bcf16ea06d6453493f62a4d857ed6ce2481a4485b", + "delphi/polismath/pca_kmeans_rep/corr.py": "00012b15a71f31a291df12e3e35c9a81b956e0e5d9a1956ebf54253645016892", + "delphi/polismath/pca_kmeans_rep/group_k_smoother.py": "c204a25f41623c9d2e64f30ee27ac260340320f50b97c164639ae204b1f47bb6", + "delphi/polismath/pca_kmeans_rep/legacy_kmeans.py": "fdb92c9fb56dff603833434911a19759fc0720ec00ee6175c8ffeb846dc6b767", + "delphi/polismath/pca_kmeans_rep/pca.py": "bb3635863a31f78387fae2b74dcf011e01c91ebd5e3aaa29a3ade55f3951e696", + "delphi/polismath/pca_kmeans_rep/repness.py": "767ed4936a18d1113d07f81e73e0da3c43926e3bb0d54103d8811e65499e54fd", + "delphi/polismath/poller/__init__.py": "444822bdc04f3acefbc3225a740689060a7482ae6e9519d9520e15f4c93471d3", + "delphi/polismath/poller/math_writer.py": "aa0da037647c9a14d962cd826e2cec50edcf48a56941cc054e3253ea1725b6c6", + "delphi/polismath/poller/service.py": "bdff3fc84f76fdfb7bfb594274884b77e4b8c6cbb6840c8243750e659406cf17", + "delphi/polismath/poller/worker_pool.py": "0a3d35f91b75768db1e9e049a8ee2fc88c5b8614b823e214d6a7f3f57812ce5b", + "delphi/polismath/regression/__init__.py": "93b935e0fd403562a661dbb75f5ff4af456441c9e7d6ec6505262e1e2a773647", + "delphi/polismath/regression/clojure_comparer.py": "b9feab2e4bda2301e515c7e8ed11d9e85332451b8f2469542facbd9aab197cb1", + "delphi/polismath/regression/comparer.py": "a98f17a03a2c513581b423965f204e006e363ba5899cb84979ef997adff8eb4a", + "delphi/polismath/regression/datasets.py": "8a6cda9f927a6769fc103b98617ef9d7d9946512c808b2d53df69289de7d7f1b", + "delphi/polismath/regression/recorder.py": "13c9b47a6f5514b64ab6df3c41c8a0cc13220c1fcdad6d007ab4e19b1e10f456", + "delphi/polismath/regression/utils.py": "b095da8892c12ac22d741aa43021d72bfa8c41901571711ef1e9612e7e81c2d8", + "delphi/polismath/replay/__init__.py": "72caea29707841d12cdbd7900ac4b0e7f9037060c212a2d2053d6528ddd9f7e7", + "delphi/polismath/replay/certify.py": "82fd3c78c6f0251823f949173df66be0374de73f7510db0c93ab1330bc13d652", + "delphi/polismath/replay/crosslang.py": "b558bcf87830d7376da7a1f635f7970ef87034ab4a262cff14a5dbb5ecc4098b", + "delphi/polismath/replay/driver.py": "d405c36751997927222cd8c92f630238fcb5c3fe0fbc8cdd428015ae4d4efa16", + "delphi/polismath/replay/fixture_bundle.py": "acdc3ffe5d6662a1280528c0c22f75bc6e8ee519289635dbff4357d7ad3d4b75", + "delphi/polismath/replay/fixture_config.py": "c97bab89066cb60b98d31c72a0c67769c38376e88b96b56f5608151c4928ce02", + "delphi/polismath/replay/fixture_extract.py": "155208de5cd59f684ff7b15fcfe7eeabeec1de880efa0b44a8f51c5c20623497", + "delphi/polismath/replay/fixture_generate.py": "4e5fc181a6ffafbea1e954ffce146ca328a85fe781479c6722c190d7b25b1344", + "delphi/polismath/replay/fixture_survey.py": "79a54ea38835327c5a4593f5b005ec2b0bc0dd1380d4dd6dbfa8d4f72e92478b", + "delphi/polismath/replay/poller_equiv.py": "6eaa5cc80c30c1983bfa0ff3e72fdf4b153d6f01cce128d618c16c45b8451986", + "delphi/polismath/replay/prodclone.py": "718a012ad266dc173d5c46338b362f186b1f13e0d0e34d28b5a7747fdbb55384", + "delphi/polismath/replay/real_data.py": "884efcd81ab82364fcf462cb55c6007414f3486e15fc5fe21be1ba9c09af0946", + "delphi/polismath/replay/schedule.py": "251092b698ab685ae9fc218406156f1745d1d86739bbfe9d1b20141379c7e656", + "delphi/polismath/replay/shard_bench.py": "9253180770096ef0fdc33ef67ea46efb3104803426b709f08e0c1d3b52c92afe", + "delphi/polismath/replay/stepcompare.py": "959b589bbf570e0d79005f4dac7056d0a4ba6b0a9f3e2cdf3ddbcd282f32e15c", + "delphi/polismath/replay/store.py": "1c837f7c32fbc8cc2cdeaa16c43a7ab025fe23fca2ad9fa6430e1376527d3b10", + "delphi/polismath/replay/types.py": "a9be8e851240af070875997848cada2fce45ea0c4fd45903d3cc12efa30622da", + "delphi/polismath/run_math_pipeline.py": "39c16083c720dfadae9f93304eeb95f2d04a852f6464bef618ff444c7ee6a069", + "delphi/polismath/utils/__init__.py": "4e1f746e69359e282c9de9a255f24feb3cbeb0c6e5564c09bbb30413178313ce", + "delphi/polismath/utils/clj_hash.py": "c81b4a39da41e04cb6cb7379a8e41d27603e5e094205397f121f6be2f58e9c99", + "delphi/polismath/utils/env_flags.py": "c3eae80667781cc9b491d673923ec947cdb9827678f653803cab0b93194a51e7", + "delphi/polismath/utils/general.py": "6199ec4cb7a038315d33583e2ef18ea1c151ff6904404ab47c5e47157e36390c", + "delphi/polismath/utils/serialization.py": "01e242cabe578ff6bf48801d4260e9be7c5619d503a45d7eed8eb0fb43d7a746", + "delphi/tests/coordinator/__init__.py": "8c61334c3002c625a92e457ffb6e37f23ace4f1fa9045d0e90b5dff269f442b0", + "delphi/tests/coordinator/audit_stages.py": "b411ed76d70b62e58d098f64a02cb4f892826452535151fb03e2be86266319a9", + "delphi/tests/coordinator/conftest.py": "a7f1ee919cb80f2709897b1c2684a3257c53965cffaafc0058d4f81557cd50af", + "delphi/tests/coordinator/reference_child.py": "add6c53bbe9bcb4b384bdb1a80b31ca51dcce0eabe1ead4d31d2b3376a25f6f5", + "delphi/tests/coordinator/test_adapter.py": "715d4cc08658df077e96842ee69008771bd33e911639f1c5e9705b8107f75ec6", + "delphi/tests/coordinator/test_equivalence.py": "a8b4978adfcba99dbdd91a805c8878cff713667771d189190ca496fcefc91670", + "delphi/tests/coordinator/test_incremental.py": "dfc47a73aab6553cdf792b7578970639726e4877ee55255f1079081366e8391b", + "delphi/tests/coordinator/test_node_reader.py": "64e52e696351113f98c3b22271b199c71c2edafd11901f84565581fd4ff5f527", + "delphi/tests/coordinator/test_recorder_controls.py": "7694f50b0796f6bf3dbe6a1e3c4477a64010af67431acde102cd867ee4118813", + "delphi/tests/coordinator/test_recovery.py": "44a1396ec474ad0a3ff00954b715d45016346313a534dc3fffd036e93e84dd44", + "delphi/tests/coordinator/test_reference_controls.py": "7d7c424359aa62e7901743f074ded8257077b8882cdba3cd09dc08f8d63c963b", + "delphi/tests/coordinator/test_s1_identity.py": "88a1b87bc5804d16b063d30270cdc7dd2528b6d03c818f5875f3c2f511bfec8a", + "delphi/tests/coordinator/test_step1_review_controls.py": "a614c81f3f5c61e21e5c0fc148e3f425a40cf881724e62588f627fd38e1986e9", + "delphi/tests/coordinator/test_step2_review_controls.py": "1901275fe35b646dfd16d37d69e72e527e32acb4e900caa8af7647cee333221c", + "delphi/tests/coordinator/test_store.py": "70688832fdb8b091ebdeecea87359b304c897973564d27ceccc599ea7329540d" + }, + "run_pins": { + "verifiable_on_producing_host_only": true, + "junit": "coordinator-rs/artifacts/s1-pytest.xml", + "python_tests": 139, + "rust_tests": { + "default": 31, + "fault": 31 + }, + "compose_project": "s1fix-e5a8", + "postgres_port": 55888, + "platform": "macOS-15.7.4-arm64-arm-64bit", + "runtime": { + "python": "3.12.11", + "packages": { + "numpy": "1.26.4", + "scipy": "1.17.1", + "pandas": "2.3.3", + "scikit-learn": "1.9.0", + "psycopg2-binary": "2.9.12", + "pytest": "9.1.1" + } + }, + "sha256": { + "coordinator-rs/artifacts/s1-cargo-default.log": "c56ecb8f8a137a84a5388a157eb227809154972b16541290a6f92b337d1ce704", + "coordinator-rs/artifacts/s1-cargo-default.status": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "coordinator-rs/artifacts/s1-cargo-fault.log": "346a7c59d8e06967311d76c135acfddd3ccd8ec477cceb96d9a57c5d3b7be1f9", + "coordinator-rs/artifacts/s1-cargo-fault.status": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "coordinator-rs/artifacts/s1-clippy-default.log": "2cba8e01a27235b416d3e1c54ca46b77be3e601cdd848f9f4e0c5670801e0d88", + "coordinator-rs/artifacts/s1-clippy-default.status": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "coordinator-rs/artifacts/s1-clippy-fault.log": "9d37a1405eaeaafcf860ad94d150cad4495c775c683052e5858e7e3f57438432", + "coordinator-rs/artifacts/s1-clippy-fault.status": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "coordinator-rs/artifacts/s1-fault-build.log": "446d404a5299191022b3bcfe5de291d913b69bc53c7177d7f5c6340befd10fde", + "coordinator-rs/artifacts/s1-fault-build.status": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "coordinator-rs/artifacts/s1-pytest.log": "05fcd3649bc998f549c332925b8bca704dbcaa029a7b0c75c0e98dcf79ae4316", + "coordinator-rs/artifacts/s1-pytest.xml": "0397abadb8fc2c47f976125b2807f062c649d45ebfe495999ad0db6a2d06948f", + "coordinator-rs/artifacts/s1-release.log": "05526c21bd3cb27709fc979c8349a96a86704db33ce70d0f223ad8013c7af42e", + "coordinator-rs/artifacts/s1-release.status": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa", + "coordinator-rs/target/fault/debug/polis-coordinator": "b081abcf885b6fff0b8f90f0931bdb224691a1678ac183066fc020058c0300dc", + "coordinator-rs/target/release/polis-coordinator": "e1a219c545bee43910d703142d0a6a97db51d5986e80f155b2850dd21c980af3" + } + } +} diff --git a/coordinator-rs/evidence/semantic-tie-key.json b/coordinator-rs/evidence/semantic-tie-key.json new file mode 100644 index 0000000000..8c78c56959 --- /dev/null +++ b/coordinator-rs/evidence/semantic-tie-key.json @@ -0,0 +1,57 @@ +{ + "declared": { + "terms": [ + { + "name": "tid", + "nulls": "default", + "source": "votes.tid" + }, + { + "name": "pid", + "nulls": "default", + "source": "votes.pid" + }, + { + "name": "created_ms", + "nulls": "default", + "source": "votes.created" + }, + { + "name": "semantic_vote", + "nulls": "default", + "source": "raw_vote * storage_agree_value" + }, + { + "name": "weight_x_32767", + "nulls": "first", + "source": "votes.weight_x_32767" + } + ], + "schema": "polis-order/1", + "profile": "live-tid-pid-created-semantic-value-weight/1", + "order_by": "tid,pid,created,(vote::bigint*$2::bigint),weight_x_32767 NULLS FIRST", + "parameter": "storage_agree_value", + "semantic_vote": "raw_vote * storage_agree_value", + "algorithm_digest": "2dc3856741cc73ad45426230758b1557b97988720b3551df9282819c5978e7d1", + "equal_time_census": { + "key": [ + "tid", + "pid", + "created_ms" + ], + "schema": "polis-census/1", + "tied_rows": 2, + "resolved_by": [ + "semantic_vote", + "weight_x_32767" + ], + "tied_groups": 1 + }, + "storage_agree_value": -1 + }, + "mirrored_digest": "3bd0700204ee8202074e31f6cfe2f57903bd9d9fc10c5149c0fcdfb7f5406685", + "a": "defe687e91d6b728d0e63e120fcc807fa2c1a84608f56159c3cc5e0ce0c7951b", + "b": "defe687e91d6b728d0e63e120fcc807fa2c1a84608f56159c3cc5e0ce0c7951b", + "negative": "5aaff63b7f65072559ae96cd7296c6341af2a2c51e93b3b39094f2b51695a3e8", + "deltas": [] +} \ No newline at end of file diff --git a/coordinator-rs/evidence/stage-inventory.json b/coordinator-rs/evidence/stage-inventory.json new file mode 100644 index 0000000000..3d1caadf7e --- /dev/null +++ b/coordinator-rs/evidence/stage-inventory.json @@ -0,0 +1,1588 @@ +{ + "protocol": "polis-fault-control/1", + "compiled_stages": 25, + "required_stages": 25, + "metrics_namespace": "Polis/Math", + "declared_metrics": 26, + "reached": 25, + "stages": [ + { + "stage": "after_source_selection", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_source_selection", + "ack": { + "context": { + "epoch": 1, + "event_count": 24, + "math_env": "rustproto", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "zid": 1 + }, + "operation_id": "184069540cd94c6eb7ab6153f4da96d1", + "pid": 67768, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_source_selection", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "after_input_checkpoint", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_input_checkpoint", + "ack": { + "context": { + "epoch": 1, + "event_count": 24, + "math_env": "rustproto", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "zid": 1 + }, + "operation_id": "8b6c911f0453472b9a2e60d6544fa94e", + "pid": 67810, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_input_checkpoint", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "before_worker_apply", + "status": "reached-and-blocked", + "evidence": { + "stage": "before_worker_apply", + "ack": { + "context": { + "fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "math_env": "rustproto", + "operation_id": "c25c383b-2df3-46d5-ab77-272bfd145880", + "worker_pid": 67866, + "zid": 1 + }, + "operation_id": "2ccfa1ea7e7c4816adc8eacf55e471b5", + "pid": 67865, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "before_worker_apply", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "after_worker_compute", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_worker_compute", + "ack": { + "context": { + "fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "math_env": "rustproto", + "operation_id": "24cad7a6-c288-49ee-ac22-657c0692e6e4", + "worker_pid": 68029, + "zid": 1 + }, + "operation_id": "5d0e49837f3442f096c88842d688bb87", + "pid": 68028, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_worker_compute", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "after_lease", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_lease", + "ack": { + "context": { + "epoch": 1, + "math_env": "rustproto", + "zid": 1 + }, + "operation_id": "5db57009d42e4542b6d37515b37d9ee9", + "pid": 67423, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_lease", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "before_ticks", + "status": "reached-and-blocked", + "evidence": { + "stage": "before_ticks", + "ack": { + "context": { + "backend_pid": 1738, + "checkpoint": { + "admission": { + "candidate_schema": "polis-candidate-input/1", + "engine_version": "python-conversation/p026-s1", + "input_digest": "2820016178151e192b3275ea16b7a11ba5b23aa03df2c30e7962500695a1d2e6", + "operation_id": "367502f2-1249-4dae-82a0-a4c12738e9d5", + "schedule_digest": "b17a696486862dd98cace0d095db54aebfe1c7953ebb97cf0c274e548c93fd24" + }, + "cursors": { + "moderation": { + "sha256": "879acd90e8236b185dc976764a60a14cc639b5e0f5cd1fef6d1c3b4f5024621b", + "slot": 1 + }, + "votes": { + "sha256": "517a4bcaeeb6a776031c63c22d86fc25bbc6486f60579098492507b464a13c5e", + "slot": 24 + } + }, + "event_count": 24, + "lifecycle": "rebuild-prefix/1", + "operation_id": "367502f2-1249-4dae-82a0-a4c12738e9d5", + "ordering": { + "algorithm_digest": "2dc3856741cc73ad45426230758b1557b97988720b3551df9282819c5978e7d1", + "equal_time_census": { + "key": [ + "tid", + "pid", + "created_ms" + ], + "resolved_by": [ + "semantic_vote", + "weight_x_32767" + ], + "schema": "polis-census/1", + "tied_groups": 0, + "tied_rows": 0 + }, + "order_by": "tid,pid,created,(vote::bigint*$2::bigint),weight_x_32767 NULLS FIRST", + "parameter": "storage_agree_value", + "profile": "live-tid-pid-created-semantic-value-weight/1", + "schema": "polis-order/1", + "semantic_vote": "raw_vote * storage_agree_value", + "storage_agree_value": -1, + "terms": [ + { + "name": "tid", + "nulls": "default", + "source": "votes.tid" + }, + { + "name": "pid", + "nulls": "default", + "source": "votes.pid" + }, + { + "name": "created_ms", + "nulls": "default", + "source": "votes.created" + }, + { + "name": "semantic_vote", + "nulls": "default", + "source": "raw_vote * storage_agree_value" + }, + { + "name": "weight_x_32767", + "nulls": "first", + "source": "votes.weight_x_32767" + } + ] + }, + "original_digests": { + "bidtopid": "686510fe4d2b41c10c6741bc311eddd0d6d1cfe9deb64c9b917bf7f654079d62", + "main": "c198ab62ffcab06e42eba252b30916b15656e17e8726e1d0721db298cc681766", + "ptptstats": "a1408b463176888e8dd41fb1c107f7333db9b7df9073d4438a9c7a102625aa5f" + }, + "payload_digests": { + "bidtopid": "bce9a00e58bfb881ee202dfb6bd7b3e8e572ffa85c9a9f230f97e4dbbeab2b78", + "main": "7bbe23b65e1884741b120c5b8042ddd3945997e726bbdf05b7c968ba8de59f87", + "ptptstats": "c71a7fdb80b86ca6f403c7f28a1b8355a702a3224587e3cc78666665ebf5c8a2" + }, + "profile": "candidate-profile", + "publisher_epoch": 1, + "schema": "polis-coordinator/1", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "storage_agree_value": -1 + }, + "epoch": 1, + "math_env": "rustproto", + "zid": 1 + }, + "operation_id": "f2777c0f13a240c88432fdd5677d1851", + "pid": 68089, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "before_ticks", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "after_ticks", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_ticks", + "ack": { + "context": { + "backend_pid": 1854, + "checkpoint": { + "admission": { + "candidate_schema": "polis-candidate-input/1", + "engine_version": "python-conversation/p026-s1", + "input_digest": "2820016178151e192b3275ea16b7a11ba5b23aa03df2c30e7962500695a1d2e6", + "operation_id": "2971ab4f-6ae3-4203-905a-1b423bae87b0", + "schedule_digest": "b17a696486862dd98cace0d095db54aebfe1c7953ebb97cf0c274e548c93fd24" + }, + "cursors": { + "moderation": { + "sha256": "879acd90e8236b185dc976764a60a14cc639b5e0f5cd1fef6d1c3b4f5024621b", + "slot": 1 + }, + "votes": { + "sha256": "517a4bcaeeb6a776031c63c22d86fc25bbc6486f60579098492507b464a13c5e", + "slot": 24 + } + }, + "event_count": 24, + "lifecycle": "rebuild-prefix/1", + "operation_id": "2971ab4f-6ae3-4203-905a-1b423bae87b0", + "ordering": { + "algorithm_digest": "2dc3856741cc73ad45426230758b1557b97988720b3551df9282819c5978e7d1", + "equal_time_census": { + "key": [ + "tid", + "pid", + "created_ms" + ], + "resolved_by": [ + "semantic_vote", + "weight_x_32767" + ], + "schema": "polis-census/1", + "tied_groups": 0, + "tied_rows": 0 + }, + "order_by": "tid,pid,created,(vote::bigint*$2::bigint),weight_x_32767 NULLS FIRST", + "parameter": "storage_agree_value", + "profile": "live-tid-pid-created-semantic-value-weight/1", + "schema": "polis-order/1", + "semantic_vote": "raw_vote * storage_agree_value", + "storage_agree_value": -1, + "terms": [ + { + "name": "tid", + "nulls": "default", + "source": "votes.tid" + }, + { + "name": "pid", + "nulls": "default", + "source": "votes.pid" + }, + { + "name": "created_ms", + "nulls": "default", + "source": "votes.created" + }, + { + "name": "semantic_vote", + "nulls": "default", + "source": "raw_vote * storage_agree_value" + }, + { + "name": "weight_x_32767", + "nulls": "first", + "source": "votes.weight_x_32767" + } + ] + }, + "original_digests": { + "bidtopid": "686510fe4d2b41c10c6741bc311eddd0d6d1cfe9deb64c9b917bf7f654079d62", + "main": "229a4bc3e8601b198c5001214ab28c1e613ea833139dda4b913e0d0b9d43bc6b", + "ptptstats": "a1408b463176888e8dd41fb1c107f7333db9b7df9073d4438a9c7a102625aa5f" + }, + "payload_digests": { + "bidtopid": "bce9a00e58bfb881ee202dfb6bd7b3e8e572ffa85c9a9f230f97e4dbbeab2b78", + "main": "81e544aa356faf3f175f4655258e665ad572fa9c11fb289421e01cfeac112781", + "ptptstats": "c71a7fdb80b86ca6f403c7f28a1b8355a702a3224587e3cc78666665ebf5c8a2" + }, + "profile": "candidate-profile", + "publisher_epoch": 1, + "schema": "polis-coordinator/1", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "storage_agree_value": -1 + }, + "epoch": 1, + "math_env": "rustproto", + "zid": 1 + }, + "operation_id": "df3b6fe34d454b639b77aa597861e45d", + "pid": 68145, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_ticks", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "before_bidtopid", + "status": "reached-and-blocked", + "evidence": { + "stage": "before_bidtopid", + "ack": { + "context": { + "backend_pid": 1969, + "checkpoint": { + "admission": { + "candidate_schema": "polis-candidate-input/1", + "engine_version": "python-conversation/p026-s1", + "input_digest": "2820016178151e192b3275ea16b7a11ba5b23aa03df2c30e7962500695a1d2e6", + "operation_id": "2d92ad9e-8dfe-480d-8e27-ff921816985c", + "schedule_digest": "b17a696486862dd98cace0d095db54aebfe1c7953ebb97cf0c274e548c93fd24" + }, + "cursors": { + "moderation": { + "sha256": "879acd90e8236b185dc976764a60a14cc639b5e0f5cd1fef6d1c3b4f5024621b", + "slot": 1 + }, + "votes": { + "sha256": "517a4bcaeeb6a776031c63c22d86fc25bbc6486f60579098492507b464a13c5e", + "slot": 24 + } + }, + "event_count": 24, + "lifecycle": "rebuild-prefix/1", + "operation_id": "2d92ad9e-8dfe-480d-8e27-ff921816985c", + "ordering": { + "algorithm_digest": "2dc3856741cc73ad45426230758b1557b97988720b3551df9282819c5978e7d1", + "equal_time_census": { + "key": [ + "tid", + "pid", + "created_ms" + ], + "resolved_by": [ + "semantic_vote", + "weight_x_32767" + ], + "schema": "polis-census/1", + "tied_groups": 0, + "tied_rows": 0 + }, + "order_by": "tid,pid,created,(vote::bigint*$2::bigint),weight_x_32767 NULLS FIRST", + "parameter": "storage_agree_value", + "profile": "live-tid-pid-created-semantic-value-weight/1", + "schema": "polis-order/1", + "semantic_vote": "raw_vote * storage_agree_value", + "storage_agree_value": -1, + "terms": [ + { + "name": "tid", + "nulls": "default", + "source": "votes.tid" + }, + { + "name": "pid", + "nulls": "default", + "source": "votes.pid" + }, + { + "name": "created_ms", + "nulls": "default", + "source": "votes.created" + }, + { + "name": "semantic_vote", + "nulls": "default", + "source": "raw_vote * storage_agree_value" + }, + { + "name": "weight_x_32767", + "nulls": "first", + "source": "votes.weight_x_32767" + } + ] + }, + "original_digests": { + "bidtopid": "686510fe4d2b41c10c6741bc311eddd0d6d1cfe9deb64c9b917bf7f654079d62", + "main": "340bb2b8ae343211f3b1d008914a91a5cbe3c6d1fc7330880d837547f5785925", + "ptptstats": "a1408b463176888e8dd41fb1c107f7333db9b7df9073d4438a9c7a102625aa5f" + }, + "payload_digests": { + "bidtopid": "bce9a00e58bfb881ee202dfb6bd7b3e8e572ffa85c9a9f230f97e4dbbeab2b78", + "main": "8dc07045e06f59787b7b09cf19aa30f39debc15a7d72b0a822517fb356fd3b67", + "ptptstats": "c71a7fdb80b86ca6f403c7f28a1b8355a702a3224587e3cc78666665ebf5c8a2" + }, + "profile": "candidate-profile", + "publisher_epoch": 1, + "schema": "polis-coordinator/1", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "storage_agree_value": -1 + }, + "epoch": 1, + "math_env": "rustproto", + "zid": 1 + }, + "operation_id": "b09d0d922958428cb623d16add95b503", + "pid": 68215, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "before_bidtopid", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "after_bidtopid", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_bidtopid", + "ack": { + "context": { + "backend_pid": 2084, + "checkpoint": { + "admission": { + "candidate_schema": "polis-candidate-input/1", + "engine_version": "python-conversation/p026-s1", + "input_digest": "2820016178151e192b3275ea16b7a11ba5b23aa03df2c30e7962500695a1d2e6", + "operation_id": "14d17f42-d27a-4f84-b3d7-5f1b5bb6a218", + "schedule_digest": "b17a696486862dd98cace0d095db54aebfe1c7953ebb97cf0c274e548c93fd24" + }, + "cursors": { + "moderation": { + "sha256": "879acd90e8236b185dc976764a60a14cc639b5e0f5cd1fef6d1c3b4f5024621b", + "slot": 1 + }, + "votes": { + "sha256": "517a4bcaeeb6a776031c63c22d86fc25bbc6486f60579098492507b464a13c5e", + "slot": 24 + } + }, + "event_count": 24, + "lifecycle": "rebuild-prefix/1", + "operation_id": "14d17f42-d27a-4f84-b3d7-5f1b5bb6a218", + "ordering": { + "algorithm_digest": "2dc3856741cc73ad45426230758b1557b97988720b3551df9282819c5978e7d1", + "equal_time_census": { + "key": [ + "tid", + "pid", + "created_ms" + ], + "resolved_by": [ + "semantic_vote", + "weight_x_32767" + ], + "schema": "polis-census/1", + "tied_groups": 0, + "tied_rows": 0 + }, + "order_by": "tid,pid,created,(vote::bigint*$2::bigint),weight_x_32767 NULLS FIRST", + "parameter": "storage_agree_value", + "profile": "live-tid-pid-created-semantic-value-weight/1", + "schema": "polis-order/1", + "semantic_vote": "raw_vote * storage_agree_value", + "storage_agree_value": -1, + "terms": [ + { + "name": "tid", + "nulls": "default", + "source": "votes.tid" + }, + { + "name": "pid", + "nulls": "default", + "source": "votes.pid" + }, + { + "name": "created_ms", + "nulls": "default", + "source": "votes.created" + }, + { + "name": "semantic_vote", + "nulls": "default", + "source": "raw_vote * storage_agree_value" + }, + { + "name": "weight_x_32767", + "nulls": "first", + "source": "votes.weight_x_32767" + } + ] + }, + "original_digests": { + "bidtopid": "686510fe4d2b41c10c6741bc311eddd0d6d1cfe9deb64c9b917bf7f654079d62", + "main": "7914246824ebf9f1380c7d6fd88e0fa96e70704fa2fcafef534beff761230cb8", + "ptptstats": "a1408b463176888e8dd41fb1c107f7333db9b7df9073d4438a9c7a102625aa5f" + }, + "payload_digests": { + "bidtopid": "bce9a00e58bfb881ee202dfb6bd7b3e8e572ffa85c9a9f230f97e4dbbeab2b78", + "main": "ba80e320962cc63608e51776e9115d638e98de33cb8f9b9638e642e0e75a178b", + "ptptstats": "c71a7fdb80b86ca6f403c7f28a1b8355a702a3224587e3cc78666665ebf5c8a2" + }, + "profile": "candidate-profile", + "publisher_epoch": 1, + "schema": "polis-coordinator/1", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "storage_agree_value": -1 + }, + "epoch": 1, + "math_env": "rustproto", + "zid": 1 + }, + "operation_id": "2ae82bc728604ae78c16b86bbb7dce72", + "pid": 68546, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_bidtopid", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "before_ptptstats", + "status": "reached-and-blocked", + "evidence": { + "stage": "before_ptptstats", + "ack": { + "context": { + "backend_pid": 2206, + "checkpoint": { + "admission": { + "candidate_schema": "polis-candidate-input/1", + "engine_version": "python-conversation/p026-s1", + "input_digest": "2820016178151e192b3275ea16b7a11ba5b23aa03df2c30e7962500695a1d2e6", + "operation_id": "65a93dba-8b4f-4357-83bd-8e6b1c01f81b", + "schedule_digest": "b17a696486862dd98cace0d095db54aebfe1c7953ebb97cf0c274e548c93fd24" + }, + "cursors": { + "moderation": { + "sha256": "879acd90e8236b185dc976764a60a14cc639b5e0f5cd1fef6d1c3b4f5024621b", + "slot": 1 + }, + "votes": { + "sha256": "517a4bcaeeb6a776031c63c22d86fc25bbc6486f60579098492507b464a13c5e", + "slot": 24 + } + }, + "event_count": 24, + "lifecycle": "rebuild-prefix/1", + "operation_id": "65a93dba-8b4f-4357-83bd-8e6b1c01f81b", + "ordering": { + "algorithm_digest": "2dc3856741cc73ad45426230758b1557b97988720b3551df9282819c5978e7d1", + "equal_time_census": { + "key": [ + "tid", + "pid", + "created_ms" + ], + "resolved_by": [ + "semantic_vote", + "weight_x_32767" + ], + "schema": "polis-census/1", + "tied_groups": 0, + "tied_rows": 0 + }, + "order_by": "tid,pid,created,(vote::bigint*$2::bigint),weight_x_32767 NULLS FIRST", + "parameter": "storage_agree_value", + "profile": "live-tid-pid-created-semantic-value-weight/1", + "schema": "polis-order/1", + "semantic_vote": "raw_vote * storage_agree_value", + "storage_agree_value": -1, + "terms": [ + { + "name": "tid", + "nulls": "default", + "source": "votes.tid" + }, + { + "name": "pid", + "nulls": "default", + "source": "votes.pid" + }, + { + "name": "created_ms", + "nulls": "default", + "source": "votes.created" + }, + { + "name": "semantic_vote", + "nulls": "default", + "source": "raw_vote * storage_agree_value" + }, + { + "name": "weight_x_32767", + "nulls": "first", + "source": "votes.weight_x_32767" + } + ] + }, + "original_digests": { + "bidtopid": "686510fe4d2b41c10c6741bc311eddd0d6d1cfe9deb64c9b917bf7f654079d62", + "main": "a49dc4d379a67277a8cf59dd2e12603653400fe3dc58f34f789a3b71bf7217ee", + "ptptstats": "a1408b463176888e8dd41fb1c107f7333db9b7df9073d4438a9c7a102625aa5f" + }, + "payload_digests": { + "bidtopid": "bce9a00e58bfb881ee202dfb6bd7b3e8e572ffa85c9a9f230f97e4dbbeab2b78", + "main": "af695d1520317cbc404818f32aca530652ec66cd10e32057ce84e08cbfe84fa6", + "ptptstats": "c71a7fdb80b86ca6f403c7f28a1b8355a702a3224587e3cc78666665ebf5c8a2" + }, + "profile": "candidate-profile", + "publisher_epoch": 1, + "schema": "polis-coordinator/1", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "storage_agree_value": -1 + }, + "epoch": 1, + "math_env": "rustproto", + "zid": 1 + }, + "operation_id": "c775e5a89f5147c3a776e3f8c8b9ee5c", + "pid": 68692, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "before_ptptstats", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "after_ptptstats", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_ptptstats", + "ack": { + "context": { + "backend_pid": 2320, + "checkpoint": { + "admission": { + "candidate_schema": "polis-candidate-input/1", + "engine_version": "python-conversation/p026-s1", + "input_digest": "2820016178151e192b3275ea16b7a11ba5b23aa03df2c30e7962500695a1d2e6", + "operation_id": "1e443369-aef7-42bd-b6b0-812873928179", + "schedule_digest": "b17a696486862dd98cace0d095db54aebfe1c7953ebb97cf0c274e548c93fd24" + }, + "cursors": { + "moderation": { + "sha256": "879acd90e8236b185dc976764a60a14cc639b5e0f5cd1fef6d1c3b4f5024621b", + "slot": 1 + }, + "votes": { + "sha256": "517a4bcaeeb6a776031c63c22d86fc25bbc6486f60579098492507b464a13c5e", + "slot": 24 + } + }, + "event_count": 24, + "lifecycle": "rebuild-prefix/1", + "operation_id": "1e443369-aef7-42bd-b6b0-812873928179", + "ordering": { + "algorithm_digest": "2dc3856741cc73ad45426230758b1557b97988720b3551df9282819c5978e7d1", + "equal_time_census": { + "key": [ + "tid", + "pid", + "created_ms" + ], + "resolved_by": [ + "semantic_vote", + "weight_x_32767" + ], + "schema": "polis-census/1", + "tied_groups": 0, + "tied_rows": 0 + }, + "order_by": "tid,pid,created,(vote::bigint*$2::bigint),weight_x_32767 NULLS FIRST", + "parameter": "storage_agree_value", + "profile": "live-tid-pid-created-semantic-value-weight/1", + "schema": "polis-order/1", + "semantic_vote": "raw_vote * storage_agree_value", + "storage_agree_value": -1, + "terms": [ + { + "name": "tid", + "nulls": "default", + "source": "votes.tid" + }, + { + "name": "pid", + "nulls": "default", + "source": "votes.pid" + }, + { + "name": "created_ms", + "nulls": "default", + "source": "votes.created" + }, + { + "name": "semantic_vote", + "nulls": "default", + "source": "raw_vote * storage_agree_value" + }, + { + "name": "weight_x_32767", + "nulls": "first", + "source": "votes.weight_x_32767" + } + ] + }, + "original_digests": { + "bidtopid": "686510fe4d2b41c10c6741bc311eddd0d6d1cfe9deb64c9b917bf7f654079d62", + "main": "a1fb0fb28a4e524be1a2024f44f0670dc4b5f5a4ff77718afda2bbe2e9f9c87f", + "ptptstats": "a1408b463176888e8dd41fb1c107f7333db9b7df9073d4438a9c7a102625aa5f" + }, + "payload_digests": { + "bidtopid": "bce9a00e58bfb881ee202dfb6bd7b3e8e572ffa85c9a9f230f97e4dbbeab2b78", + "main": "0790453e270310c6115a03c40d221f2214f9b13a6f6d8dcefb866aab29e81e3d", + "ptptstats": "c71a7fdb80b86ca6f403c7f28a1b8355a702a3224587e3cc78666665ebf5c8a2" + }, + "profile": "candidate-profile", + "publisher_epoch": 1, + "schema": "polis-coordinator/1", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "storage_agree_value": -1 + }, + "epoch": 1, + "math_env": "rustproto", + "zid": 1 + }, + "operation_id": "be25470fb0ac4a1fb0ebd00ae77d333b", + "pid": 68730, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_ptptstats", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "before_main", + "status": "reached-and-blocked", + "evidence": { + "stage": "before_main", + "ack": { + "context": { + "backend_pid": 2435, + "checkpoint": { + "admission": { + "candidate_schema": "polis-candidate-input/1", + "engine_version": "python-conversation/p026-s1", + "input_digest": "2820016178151e192b3275ea16b7a11ba5b23aa03df2c30e7962500695a1d2e6", + "operation_id": "919f92dc-257f-40ab-bb2f-7fbe1bcaf509", + "schedule_digest": "b17a696486862dd98cace0d095db54aebfe1c7953ebb97cf0c274e548c93fd24" + }, + "cursors": { + "moderation": { + "sha256": "879acd90e8236b185dc976764a60a14cc639b5e0f5cd1fef6d1c3b4f5024621b", + "slot": 1 + }, + "votes": { + "sha256": "517a4bcaeeb6a776031c63c22d86fc25bbc6486f60579098492507b464a13c5e", + "slot": 24 + } + }, + "event_count": 24, + "lifecycle": "rebuild-prefix/1", + "operation_id": "919f92dc-257f-40ab-bb2f-7fbe1bcaf509", + "ordering": { + "algorithm_digest": "2dc3856741cc73ad45426230758b1557b97988720b3551df9282819c5978e7d1", + "equal_time_census": { + "key": [ + "tid", + "pid", + "created_ms" + ], + "resolved_by": [ + "semantic_vote", + "weight_x_32767" + ], + "schema": "polis-census/1", + "tied_groups": 0, + "tied_rows": 0 + }, + "order_by": "tid,pid,created,(vote::bigint*$2::bigint),weight_x_32767 NULLS FIRST", + "parameter": "storage_agree_value", + "profile": "live-tid-pid-created-semantic-value-weight/1", + "schema": "polis-order/1", + "semantic_vote": "raw_vote * storage_agree_value", + "storage_agree_value": -1, + "terms": [ + { + "name": "tid", + "nulls": "default", + "source": "votes.tid" + }, + { + "name": "pid", + "nulls": "default", + "source": "votes.pid" + }, + { + "name": "created_ms", + "nulls": "default", + "source": "votes.created" + }, + { + "name": "semantic_vote", + "nulls": "default", + "source": "raw_vote * storage_agree_value" + }, + { + "name": "weight_x_32767", + "nulls": "first", + "source": "votes.weight_x_32767" + } + ] + }, + "original_digests": { + "bidtopid": "686510fe4d2b41c10c6741bc311eddd0d6d1cfe9deb64c9b917bf7f654079d62", + "main": "e1f94d1ee83f7fb8f7b3ad17a171e1bfeb1fd452f5d642967cff30238639f2d9", + "ptptstats": "a1408b463176888e8dd41fb1c107f7333db9b7df9073d4438a9c7a102625aa5f" + }, + "payload_digests": { + "bidtopid": "bce9a00e58bfb881ee202dfb6bd7b3e8e572ffa85c9a9f230f97e4dbbeab2b78", + "main": "e6d3e4a4531f7d8e5cec5976bcefa00d8557a296b8dbd1418180ece299e406a1", + "ptptstats": "c71a7fdb80b86ca6f403c7f28a1b8355a702a3224587e3cc78666665ebf5c8a2" + }, + "profile": "candidate-profile", + "publisher_epoch": 1, + "schema": "polis-coordinator/1", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "storage_agree_value": -1 + }, + "epoch": 1, + "math_env": "rustproto", + "zid": 1 + }, + "operation_id": "06e25dbf6da744b2993be0eb90a7e1d1", + "pid": 68801, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "before_main", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "after_main", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_main", + "ack": { + "context": { + "backend_pid": 2549, + "checkpoint": { + "admission": { + "candidate_schema": "polis-candidate-input/1", + "engine_version": "python-conversation/p026-s1", + "input_digest": "2820016178151e192b3275ea16b7a11ba5b23aa03df2c30e7962500695a1d2e6", + "operation_id": "79fbbaad-c7b6-4aa2-ae43-16a887a26912", + "schedule_digest": "b17a696486862dd98cace0d095db54aebfe1c7953ebb97cf0c274e548c93fd24" + }, + "cursors": { + "moderation": { + "sha256": "879acd90e8236b185dc976764a60a14cc639b5e0f5cd1fef6d1c3b4f5024621b", + "slot": 1 + }, + "votes": { + "sha256": "517a4bcaeeb6a776031c63c22d86fc25bbc6486f60579098492507b464a13c5e", + "slot": 24 + } + }, + "event_count": 24, + "lifecycle": "rebuild-prefix/1", + "operation_id": "79fbbaad-c7b6-4aa2-ae43-16a887a26912", + "ordering": { + "algorithm_digest": "2dc3856741cc73ad45426230758b1557b97988720b3551df9282819c5978e7d1", + "equal_time_census": { + "key": [ + "tid", + "pid", + "created_ms" + ], + "resolved_by": [ + "semantic_vote", + "weight_x_32767" + ], + "schema": "polis-census/1", + "tied_groups": 0, + "tied_rows": 0 + }, + "order_by": "tid,pid,created,(vote::bigint*$2::bigint),weight_x_32767 NULLS FIRST", + "parameter": "storage_agree_value", + "profile": "live-tid-pid-created-semantic-value-weight/1", + "schema": "polis-order/1", + "semantic_vote": "raw_vote * storage_agree_value", + "storage_agree_value": -1, + "terms": [ + { + "name": "tid", + "nulls": "default", + "source": "votes.tid" + }, + { + "name": "pid", + "nulls": "default", + "source": "votes.pid" + }, + { + "name": "created_ms", + "nulls": "default", + "source": "votes.created" + }, + { + "name": "semantic_vote", + "nulls": "default", + "source": "raw_vote * storage_agree_value" + }, + { + "name": "weight_x_32767", + "nulls": "first", + "source": "votes.weight_x_32767" + } + ] + }, + "original_digests": { + "bidtopid": "686510fe4d2b41c10c6741bc311eddd0d6d1cfe9deb64c9b917bf7f654079d62", + "main": "751dc135ced20d47b9ebde35cc915711b4479a0d8fe8322ad3bd42a8098c5842", + "ptptstats": "a1408b463176888e8dd41fb1c107f7333db9b7df9073d4438a9c7a102625aa5f" + }, + "payload_digests": { + "bidtopid": "bce9a00e58bfb881ee202dfb6bd7b3e8e572ffa85c9a9f230f97e4dbbeab2b78", + "main": "1d1045bfde8e59e272bbfd3dba4236bb5fc7cec3f018defc2ac0e9b320fa6c45", + "ptptstats": "c71a7fdb80b86ca6f403c7f28a1b8355a702a3224587e3cc78666665ebf5c8a2" + }, + "profile": "candidate-profile", + "publisher_epoch": 1, + "schema": "polis-coordinator/1", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "storage_agree_value": -1 + }, + "epoch": 1, + "math_env": "rustproto", + "zid": 1 + }, + "operation_id": "de6960a0f4de478ca55346beb8517444", + "pid": 69108, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_main", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "before_commit", + "status": "reached-and-blocked", + "evidence": { + "stage": "before_commit", + "ack": { + "context": { + "backend_pid": 2663, + "checkpoint": { + "admission": { + "candidate_schema": "polis-candidate-input/1", + "engine_version": "python-conversation/p026-s1", + "input_digest": "2820016178151e192b3275ea16b7a11ba5b23aa03df2c30e7962500695a1d2e6", + "operation_id": "cefe2a05-316b-4ce9-a167-e89e64b56271", + "schedule_digest": "b17a696486862dd98cace0d095db54aebfe1c7953ebb97cf0c274e548c93fd24" + }, + "cursors": { + "moderation": { + "sha256": "879acd90e8236b185dc976764a60a14cc639b5e0f5cd1fef6d1c3b4f5024621b", + "slot": 1 + }, + "votes": { + "sha256": "517a4bcaeeb6a776031c63c22d86fc25bbc6486f60579098492507b464a13c5e", + "slot": 24 + } + }, + "event_count": 24, + "lifecycle": "rebuild-prefix/1", + "operation_id": "cefe2a05-316b-4ce9-a167-e89e64b56271", + "ordering": { + "algorithm_digest": "2dc3856741cc73ad45426230758b1557b97988720b3551df9282819c5978e7d1", + "equal_time_census": { + "key": [ + "tid", + "pid", + "created_ms" + ], + "resolved_by": [ + "semantic_vote", + "weight_x_32767" + ], + "schema": "polis-census/1", + "tied_groups": 0, + "tied_rows": 0 + }, + "order_by": "tid,pid,created,(vote::bigint*$2::bigint),weight_x_32767 NULLS FIRST", + "parameter": "storage_agree_value", + "profile": "live-tid-pid-created-semantic-value-weight/1", + "schema": "polis-order/1", + "semantic_vote": "raw_vote * storage_agree_value", + "storage_agree_value": -1, + "terms": [ + { + "name": "tid", + "nulls": "default", + "source": "votes.tid" + }, + { + "name": "pid", + "nulls": "default", + "source": "votes.pid" + }, + { + "name": "created_ms", + "nulls": "default", + "source": "votes.created" + }, + { + "name": "semantic_vote", + "nulls": "default", + "source": "raw_vote * storage_agree_value" + }, + { + "name": "weight_x_32767", + "nulls": "first", + "source": "votes.weight_x_32767" + } + ] + }, + "original_digests": { + "bidtopid": "686510fe4d2b41c10c6741bc311eddd0d6d1cfe9deb64c9b917bf7f654079d62", + "main": "67e0a5b61c00007b440e3501cf145577f5ea18a42ec0efd07d563e8a82288193", + "ptptstats": "a1408b463176888e8dd41fb1c107f7333db9b7df9073d4438a9c7a102625aa5f" + }, + "payload_digests": { + "bidtopid": "bce9a00e58bfb881ee202dfb6bd7b3e8e572ffa85c9a9f230f97e4dbbeab2b78", + "main": "8e59f38fc2f7a205e390569dd3212f231236e54b60124bd88b04ce342c3c5988", + "ptptstats": "c71a7fdb80b86ca6f403c7f28a1b8355a702a3224587e3cc78666665ebf5c8a2" + }, + "profile": "candidate-profile", + "publisher_epoch": 1, + "schema": "polis-coordinator/1", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "storage_agree_value": -1 + }, + "epoch": 1, + "math_env": "rustproto", + "zid": 1 + }, + "operation_id": "f2a7b5d24a794693a0b0c6dad17e2034", + "pid": 69179, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "before_commit", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "after_commit", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_commit", + "ack": { + "context": { + "backend_pid": 2776, + "checkpoint": { + "admission": { + "candidate_schema": "polis-candidate-input/1", + "engine_version": "python-conversation/p026-s1", + "input_digest": "2820016178151e192b3275ea16b7a11ba5b23aa03df2c30e7962500695a1d2e6", + "operation_id": "612592ab-ce21-4473-829c-00cd2e9cf130", + "schedule_digest": "b17a696486862dd98cace0d095db54aebfe1c7953ebb97cf0c274e548c93fd24" + }, + "cursors": { + "moderation": { + "sha256": "879acd90e8236b185dc976764a60a14cc639b5e0f5cd1fef6d1c3b4f5024621b", + "slot": 1 + }, + "votes": { + "sha256": "517a4bcaeeb6a776031c63c22d86fc25bbc6486f60579098492507b464a13c5e", + "slot": 24 + } + }, + "event_count": 24, + "lifecycle": "rebuild-prefix/1", + "operation_id": "612592ab-ce21-4473-829c-00cd2e9cf130", + "ordering": { + "algorithm_digest": "2dc3856741cc73ad45426230758b1557b97988720b3551df9282819c5978e7d1", + "equal_time_census": { + "key": [ + "tid", + "pid", + "created_ms" + ], + "resolved_by": [ + "semantic_vote", + "weight_x_32767" + ], + "schema": "polis-census/1", + "tied_groups": 0, + "tied_rows": 0 + }, + "order_by": "tid,pid,created,(vote::bigint*$2::bigint),weight_x_32767 NULLS FIRST", + "parameter": "storage_agree_value", + "profile": "live-tid-pid-created-semantic-value-weight/1", + "schema": "polis-order/1", + "semantic_vote": "raw_vote * storage_agree_value", + "storage_agree_value": -1, + "terms": [ + { + "name": "tid", + "nulls": "default", + "source": "votes.tid" + }, + { + "name": "pid", + "nulls": "default", + "source": "votes.pid" + }, + { + "name": "created_ms", + "nulls": "default", + "source": "votes.created" + }, + { + "name": "semantic_vote", + "nulls": "default", + "source": "raw_vote * storage_agree_value" + }, + { + "name": "weight_x_32767", + "nulls": "first", + "source": "votes.weight_x_32767" + } + ] + }, + "original_digests": { + "bidtopid": "686510fe4d2b41c10c6741bc311eddd0d6d1cfe9deb64c9b917bf7f654079d62", + "main": "f6cc744e32cf82c66cacc1bded0877f73f2b2aad0e7e20ad2b13ea8ae0c18492", + "ptptstats": "a1408b463176888e8dd41fb1c107f7333db9b7df9073d4438a9c7a102625aa5f" + }, + "payload_digests": { + "bidtopid": "bce9a00e58bfb881ee202dfb6bd7b3e8e572ffa85c9a9f230f97e4dbbeab2b78", + "main": "3fd3b4d2bd57d0ba6b04f9e62bddafdbd6df94dde2e18b9edb61aa558d8dcbe9", + "ptptstats": "c71a7fdb80b86ca6f403c7f28a1b8355a702a3224587e3cc78666665ebf5c8a2" + }, + "profile": "candidate-profile", + "publisher_epoch": 1, + "schema": "polis-coordinator/1", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "storage_agree_value": -1 + }, + "epoch": 1, + "math_env": "rustproto", + "zid": 1 + }, + "operation_id": "3ac657abfb4848729ef4da502bcc7eb6", + "pid": 69251, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_commit", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "after_ack", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_ack", + "ack": { + "context": { + "epoch": 1, + "event_count": 24, + "math_env": "rustproto", + "source_fingerprint": "9d9f34f05fea81bed7f90bfe641e5c5ca4a833d41f9830e70871353bd6bce57f", + "zid": 1 + }, + "operation_id": "e9df8846114a4f68a6afeaea47a46768", + "pid": 69388, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_ack", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "exit": -9, + "restart": "passed", + "manual_lease_expiry": false, + "repaired_epoch": 2, + "math_tick": 0, + "postcondition": "coherent; independent fold exact; dead owner's lease never expired by the harness" + } + }, + { + "stage": "before_restore", + "status": "reached-and-blocked", + "evidence": { + "stage": "before_restore", + "ack": { + "context": { + "fingerprint": "c96dd4fdf8ca7b7b09176eae1abf6fd32adbacd0378b24cf46c9b250e0be32be", + "math_env": "rustproto", + "operation_id": "7f32044a-176f-4964-985f-c890cf86dc8d", + "worker_pid": 70201, + "zid": 1 + }, + "operation_id": "d2c0f57fe7254241a52f08f83a08aa50", + "pid": 70199, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "before_restore", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "restart": "passed", + "manual_lease_expiry": false, + "postcondition": "coherent independent fold" + } + }, + { + "stage": "after_restore", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_restore", + "ack": { + "context": { + "fingerprint": "c96dd4fdf8ca7b7b09176eae1abf6fd32adbacd0378b24cf46c9b250e0be32be", + "math_env": "rustproto", + "operation_id": "c48456a9-fcff-4663-aa0e-8218fb2d4d65", + "worker_pid": 70347, + "zid": 1 + }, + "operation_id": "90637e6318a44a10a020bd763d129fc6", + "pid": 70346, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_restore", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "restart": "passed", + "manual_lease_expiry": false, + "postcondition": "coherent independent fold" + } + }, + { + "stage": "before_cursor", + "status": "reached-and-blocked", + "evidence": { + "stage": "before_cursor", + "ack": { + "context": {}, + "operation_id": "d5fcb22ce3fa4f6b8ffc7d92b0661d52", + "pid": 70409, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "before_cursor", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "restart": "passed", + "postcondition": "reader redelivered complete bundle" + } + }, + { + "stage": "after_cursor", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_cursor", + "ack": { + "context": { + "after_tick": -1, + "after_zid": 0, + "high": 1, + "maximum": 1, + "sweep": 0 + }, + "operation_id": "1603e1a3216e4070b043e35c8e2dbb4b", + "pid": 70424, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_cursor", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "restart": "passed", + "postcondition": "reader redelivered complete bundle" + } + }, + { + "stage": "before_sweep", + "status": "reached-and-blocked", + "evidence": { + "stage": "before_sweep", + "ack": { + "context": {}, + "operation_id": "e394ca784809476faa3f07c2455dba97", + "pid": 70446, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "before_sweep", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "restart": "passed", + "postcondition": "reader redelivered complete bundle" + } + }, + { + "stage": "after_sweep", + "status": "reached-and-blocked", + "evidence": { + "stage": "after_sweep", + "ack": { + "context": { + "next_sweep": 0, + "rows": [ + { + "caching_tick": 1, + "math_tick": 0, + "zid": 1 + } + ] + }, + "operation_id": "6e4f667f019f46c6852b9154bb35220a", + "pid": 70475, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "after_sweep", + "state": "reached-and-blocked" + }, + "signal": "SIGKILL", + "restart": "passed", + "postcondition": "reader redelivered complete bundle" + } + }, + { + "stage": "bundle_pinned", + "status": "reached-and-blocked", + "evidence": { + "stage": "bundle_pinned", + "ack": { + "context": { + "tick": 0, + "zid": 1 + }, + "operation_id": "2c5973f9dcb54412af4e13ec18eb1131", + "pid": 70514, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "bundle_pinned", + "state": "reached-and-blocked" + }, + "action": "atomic replacement by second process; release", + "postcondition": "old pinned bundle remains coherent" + } + }, + { + "stage": "before_companion_join", + "status": "reached-and-blocked", + "evidence": { + "stage": "before_companion_join", + "ack": { + "context": { + "tick": 0, + "zid": 1 + }, + "operation_id": "c2a18ab6369040d08b671404fa556e4d", + "pid": 70553, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "before_companion_join", + "state": "reached-and-blocked" + }, + "action": "atomic replacement by second process; release", + "postcondition": "old pinned bundle remains coherent" + } + }, + { + "stage": "cache_eviction_contends_with_same_zid_update", + "status": "reached-and-blocked", + "evidence": { + "stage": "cache_eviction_contends_with_same_zid_update", + "ack": { + "context": { + "cache_len": 1, + "capacity": 1, + "evicted_zid": 1, + "inserted_zid": 2 + }, + "operation_id": "ce3cead6b2a14b0f96afc3f8b5b4bf03", + "pid": 69803, + "protocol": "polis-fault-control/1", + "run_id": "synthetic-test", + "stage": "cache_eviction_contends_with_same_zid_update", + "state": "reached-and-blocked" + }, + "action": "same-zid vote committed while the LRU eviction of that zid is blocked", + "postcondition": "no lost input; evicted zid republished coherently; warm output equals the cold-rebuild reference" + } + } + ], + "unreached": [], + "stage_inventory_gate": "PASS", + "open_conditions": [ + "CO08/D4: the real Node reader (getPca + getBidIndexToPidMapping + getPidsForGid) serves identical Rust/Python bytes at generation one, and the real pca2 route serves generation zero, but CO04's loadBundle/Bundle cache-unit rewrite does not exist in the server and is untested here", + "CO08/D4: application boot, auth, report.ts/doFamousQuery and the private 2,884-case served corpus are not executed; the empty presentation is compared synthetically", + "CO08/D4 reader defect (server owner): getPca(zid, undefined) misses a cold committed generation zero and finds it once the route warms the cache; the route itself, which passes math_tick=-1, serves it with 200 and ETag \"0\"", + "C7 / polis-empty-served/1: the Python comparison writer publishes no row for a zero-vote conversation, the server synthesizes the approved-comment listing with a request-clock lastVoteTimestamp, and a published empty generation lists no tids; the transition must preserve comment ownership; the contract owner must still admit a frozen clock or path-specific clock normalization before this gate closes", + "CO01: the incremental probe is a hint bounded by P026_RECONCILE_SECONDS, measured from before the source read; the interval makes a conversation eligible for reconciliation and is not a deadline, and no pass/service budget has been measured", + "CO02/CO06: persisted payloads become eligible for revalidation once the ceiling elapses, not on every pass; resident-cache reconciliation is single-threaded and no multi-worker or warm-worker cache campaign exists", + "P-031: this crate implements none of A01 PollHealthy, A02 PublishLagSeconds or A03 ObserverHealthy, has no deployed publisher and no delivery proof; the optional synchronous sink can still block when enabled", + "O8 remains OPEN and is at most PARTIAL: polis-candidate-input/1 is claimed locally as a candidate profile, not a G01-G16 certificate. S1 records the distinct candidate schema id, strict adapter admission, original-byte digests with decoded correspondence, and the worker/engine pins; still outstanding are all applicable G01-G16 cases with their required negative controls, an immutable manifest, the distinction between fresh rebuild, exact resume and warm incremental output schedules, and a non-vacuous P-023 compensated pair with raw C9 validation" + ], + "condition_states": { + "O1": "OPEN", + "O2": "OPEN", + "O3": "OPEN", + "O4": "OPEN", + "O5": "OPEN", + "O6": "OPEN", + "O7": "OPEN", + "O8": "PARTIAL (S1 identity/custody recorded; G01-G16 open)" + }, + "closed_conditions": [], + "partial_conditions": [ + { + "id": "O8", + "state": "PARTIAL", + "standing_condition": "O8 remains OPEN and is at most PARTIAL: polis-candidate-input/1 is claimed locally as a candidate profile, not a G01-G16 certificate. S1 records the distinct candidate schema id, strict adapter admission, original-byte digests with decoded correspondence, and the worker/engine pins; still outstanding are all applicable G01-G16 cases with their required negative controls, an immutable manifest, the distinction between fresh rebuild, exact resume and warm incremental output schedules, and a non-vacuous P-023 compensated pair with raw C9 validation", + "evidence": "coordinator-rs/evidence/s1-closure.json", + "scope": "P-026 step-4 S1 candidate identity and original-byte custody", + "recorded": [ + "distinct candidate schema id and strict adapter admission on both sides", + "original worker bytes and their digests stored atomically with the JSONB, with decoded original/JSONB correspondence checked on read", + "worker/engine identity re-verified at initialization and at snapshot", + "operation id and publisher epoch bound into the uncertain-COMMIT readback" + ], + "remaining_obligations": [ + "all applicable G01-G16 cases and their required negative controls", + "an immutable manifest for the profile", + "fresh rebuild, exact resume and warm incremental output schedules distinguished", + "a non-vacuous P-023 compensated pair and raw C9 validation" + ], + "g01_g16_certified": false, + "run_pins_verified_here": false + } + ], + "full_contract_gate": "FAIL", + "profile": "Rust coordinator + Python worker + CLI Bundle reader, the real Node getPca/getBidIndexToPidMapping reader in-process, and the real pca2 route over loopback HTTP; no loadBundle, no application boot" +} diff --git a/coordinator-rs/evidence/test-summary.json b/coordinator-rs/evidence/test-summary.json new file mode 100644 index 0000000000..df5754abcd --- /dev/null +++ b/coordinator-rs/evidence/test-summary.json @@ -0,0 +1,99 @@ +{ + "run": "P-026 step 4 / S1", + "python_full_suite": { + "passed": 139, + "failed": 0, + "errors": 0, + "skipped": 0, + "by_module": { + "test_adapter": 31, + "test_equivalence": 6, + "test_incremental": 6, + "test_node_reader": 2, + "test_recorder_controls": 8, + "test_recovery": 51, + "test_reference_controls": 2, + "test_s1_identity": 15, + "test_step1_review_controls": 6, + "test_step2_review_controls": 6, + "test_store": 6 + }, + "junit": "coordinator-rs/artifacts/s1-pytest.xml" + }, + "rust_tests": { + "default": 31, + "fault": 31 + }, + "clippy": "both feature sets: PASS --all-targets -D warnings", + "builds": { + "release": "PASS", + "debug_fault_injection": "PASS", + "release_fault_injection": "EXPECTED REFUSAL asserted in Python suite" + }, + "stage_audit": { + "reached": 25, + "stage_inventory_gate": "PASS", + "full_contract_gate": "FAIL", + "open_conditions": [ + "CO08/D4: the real Node reader (getPca + getBidIndexToPidMapping + getPidsForGid) serves identical Rust/Python bytes at generation one, and the real pca2 route serves generation zero, but CO04's loadBundle/Bundle cache-unit rewrite does not exist in the server and is untested here", + "CO08/D4: application boot, auth, report.ts/doFamousQuery and the private 2,884-case served corpus are not executed; the empty presentation is compared synthetically", + "CO08/D4 reader defect (server owner): getPca(zid, undefined) misses a cold committed generation zero and finds it once the route warms the cache; the route itself, which passes math_tick=-1, serves it with 200 and ETag \"0\"", + "C7 / polis-empty-served/1: the Python comparison writer publishes no row for a zero-vote conversation, the server synthesizes the approved-comment listing with a request-clock lastVoteTimestamp, and a published empty generation lists no tids; the transition must preserve comment ownership; the contract owner must still admit a frozen clock or path-specific clock normalization before this gate closes", + "CO01: the incremental probe is a hint bounded by P026_RECONCILE_SECONDS, measured from before the source read; the interval makes a conversation eligible for reconciliation and is not a deadline, and no pass/service budget has been measured", + "CO02/CO06: persisted payloads become eligible for revalidation once the ceiling elapses, not on every pass; resident-cache reconciliation is single-threaded and no multi-worker or warm-worker cache campaign exists", + "P-031: this crate implements none of A01 PollHealthy, A02 PublishLagSeconds or A03 ObserverHealthy, has no deployed publisher and no delivery proof; the optional synchronous sink can still block when enabled", + "O8 remains OPEN and is at most PARTIAL: polis-candidate-input/1 is claimed locally as a candidate profile, not a G01-G16 certificate. S1 records the distinct candidate schema id, strict adapter admission, original-byte digests with decoded correspondence, and the worker/engine pins; still outstanding are all applicable G01-G16 cases with their required negative controls, an immutable manifest, the distinction between fresh rebuild, exact resume and warm incremental output schedules, and a non-vacuous P-023 compensated pair with raw C9 validation" + ], + "condition_states": { + "O1": "OPEN", + "O2": "OPEN", + "O3": "OPEN", + "O4": "OPEN", + "O5": "OPEN", + "O6": "OPEN", + "O7": "OPEN", + "O8": "PARTIAL (S1 identity/custody recorded; G01-G16 open)" + }, + "closed_conditions": [], + "partial_conditions": [ + { + "id": "O8", + "state": "PARTIAL", + "standing_condition": "O8 remains OPEN and is at most PARTIAL: polis-candidate-input/1 is claimed locally as a candidate profile, not a G01-G16 certificate. S1 records the distinct candidate schema id, strict adapter admission, original-byte digests with decoded correspondence, and the worker/engine pins; still outstanding are all applicable G01-G16 cases with their required negative controls, an immutable manifest, the distinction between fresh rebuild, exact resume and warm incremental output schedules, and a non-vacuous P-023 compensated pair with raw C9 validation", + "evidence": "coordinator-rs/evidence/s1-closure.json", + "scope": "P-026 step-4 S1 candidate identity and original-byte custody", + "recorded": [ + "distinct candidate schema id and strict adapter admission on both sides", + "original worker bytes and their digests stored atomically with the JSONB, with decoded original/JSONB correspondence checked on read", + "worker/engine identity re-verified at initialization and at snapshot", + "operation id and publisher epoch bound into the uncertain-COMMIT readback" + ], + "remaining_obligations": [ + "all applicable G01-G16 cases and their required negative controls", + "an immutable manifest for the profile", + "fresh rebuild, exact resume and warm incremental output schedules distinguished", + "a non-vacuous P-023 compensated pair and raw C9 validation" + ], + "g01_g16_certified": false, + "run_pins_verified_here": false + } + ] + }, + "replay_comparison": { + "checkpoints": 3, + "deltas": [ + 0, + 0, + 0 + ], + "hashes_identical_to_previous_evidence": true, + "observer_errors": 0, + "observations": 200 + }, + "polarity": "four artifacts and semantic tie control unchanged; compensated pairs equal, negative controls differ", + "node_d4": "0 differences; generation-zero route 200 / ETag 0 / conditional 304; no loadBundle claim", + "scope": "Rust versus Python rebuild-prefix campaign; Clojure remains the live production writer; no Rust/Clojure comparison", + "compose_project": "s1fix-e5a8", + "postgres_port": 55888, + "uncommitted": false +} diff --git a/coordinator-rs/evidence/vw-equivalence.json b/coordinator-rs/evidence/vw-equivalence.json new file mode 100644 index 0000000000..ba73e20d4b --- /dev/null +++ b/coordinator-rs/evidence/vw-equivalence.json @@ -0,0 +1,55 @@ +{ + "checkpoints": [ + { + "cut": 1170, + "tick": 0, + "rust": "47a2b67b7717dd75c9724f90a21602af595726942405d95f5b03f3a4e2ef1130", + "python": "47a2b67b7717dd75c9724f90a21602af595726942405d95f5b03f3a4e2ef1130", + "deltas": [], + "fold_errors": { + "rustproto": [], + "python": [] + }, + "unclustered_participants": { + "rustproto": 1, + "python": 1 + }, + "fold_scope": "all-voter counts/timestamps/events plus independent latest-cell A/D/S per emitted bucket; no independent eligibility certificate" + }, + { + "cut": 2341, + "tick": 1, + "rust": "ff191bd0dca54c19ca5f92bde6bf1bd5c6ccfd7743e13e2b00707e313bfbaf6a", + "python": "ff191bd0dca54c19ca5f92bde6bf1bd5c6ccfd7743e13e2b00707e313bfbaf6a", + "deltas": [], + "fold_errors": { + "rustproto": [], + "python": [] + }, + "unclustered_participants": { + "rustproto": 9, + "python": 9 + }, + "fold_scope": "all-voter counts/timestamps/events plus independent latest-cell A/D/S per emitted bucket; no independent eligibility certificate" + }, + { + "cut": 4683, + "tick": 2, + "rust": "bcaa4dcaf378e1243a4ac16bc8543bb0133e2e0386e200d9fc297b9f3c4680dc", + "python": "bcaa4dcaf378e1243a4ac16bc8543bb0133e2e0386e200d9fc297b9f3c4680dc", + "deltas": [], + "fold_errors": { + "rustproto": [], + "python": [] + }, + "unclustered_participants": { + "rustproto": 2, + "python": 2 + }, + "fold_scope": "all-voter counts/timestamps/events plus independent latest-cell A/D/S per emitted bucket; no independent eligibility certificate" + } + ], + "observer_errors": [], + "observations": 200, + "profile": "rebuild-prefix at each checkpoint; static unmoderated comment snapshot" +} \ No newline at end of file diff --git a/coordinator-rs/migration.sql b/coordinator-rs/migration.sql new file mode 100644 index 0000000000..a699ae68ea --- /dev/null +++ b/coordinator-rs/migration.sql @@ -0,0 +1,43 @@ +-- Prototype-only controlled transition, with all writers stopped. The Python +-- comparator uses a distinct namespace; never mix MAX+1 and sequence in one env. +BEGIN; +ALTER TABLE math_ticks ADD COLUMN IF NOT EXISTS publisher_epoch bigint; +ALTER TABLE math_ticks ADD COLUMN IF NOT EXISTS input_checkpoint jsonb; +ALTER TABLE math_ticks ADD COLUMN IF NOT EXISTS operation_id text; +-- Nullable for legacy rows. Rust requires all originals and digests on admission; +-- a controlled migration never invents the bytes that JSONB has already lost. +ALTER TABLE math_main ADD COLUMN IF NOT EXISTS original_bytes bytea; +ALTER TABLE math_main ADD COLUMN IF NOT EXISTS original_sha256 text; +ALTER TABLE math_bidtopid ADD COLUMN IF NOT EXISTS original_bytes bytea; +ALTER TABLE math_bidtopid ADD COLUMN IF NOT EXISTS original_sha256 text; +ALTER TABLE math_ptptstats ADD COLUMN IF NOT EXISTS original_bytes bytea; +ALTER TABLE math_ptptstats ADD COLUMN IF NOT EXISTS original_sha256 text; +CREATE TABLE IF NOT EXISTS coordinator_leases ( + math_env varchar(999) NOT NULL, zid integer NOT NULL REFERENCES conversations(zid), + owner_id text NOT NULL, owner_epoch bigint NOT NULL CHECK(owner_epoch>0), + expires_at timestamptz NOT NULL, PRIMARY KEY(math_env,zid) +); +CREATE TABLE IF NOT EXISTS coordinator_cursors ( + math_env varchar(999) NOT NULL, consumer text NOT NULL, position jsonb NOT NULL, + PRIMARY KEY(math_env,consumer) +); +CREATE TABLE IF NOT EXISTS coordinator_failures ( + math_env varchar(999) NOT NULL, zid integer NOT NULL REFERENCES conversations(zid), + attempts integer NOT NULL, first_failed_at timestamptz NOT NULL DEFAULT clock_timestamp(), + next_attempt timestamptz NOT NULL, PRIMARY KEY(math_env,zid) +); +-- CO01 incremental discovery cursor: when the authoritative full snapshot was +-- last taken for this conversation, and the cheap probe observed immediately +-- before it. The probe is a hint that may skip a full read; `reconciled_at` is +-- what bounds how long it may be trusted, and it is the source of the +-- OldestReconciliationAgeSeconds scan-age metric. +CREATE TABLE IF NOT EXISTS coordinator_reconciliation ( + math_env varchar(999) NOT NULL, zid integer NOT NULL REFERENCES conversations(zid), + reconciled_at timestamptz NOT NULL, source_probe jsonb NOT NULL, + PRIMARY KEY(math_env,zid) +); +CREATE SEQUENCE IF NOT EXISTS coordinator_caching_tick; +SELECT setval('coordinator_caching_tick', GREATEST( + (SELECT COALESCE(MAX(caching_tick),0)+1 FROM math_main), + (SELECT last_value + CASE WHEN is_called THEN 1 ELSE 0 END FROM coordinator_caching_tick)), false); +COMMIT; diff --git a/coordinator-rs/rust-toolchain.toml b/coordinator-rs/rust-toolchain.toml new file mode 100644 index 0000000000..273a2d7e55 --- /dev/null +++ b/coordinator-rs/rust-toolchain.toml @@ -0,0 +1,6 @@ +# Pinned so every build of this crate uses one known toolchain. +# Must satisfy the crate's `rust-version` (MSRV) in Cargo.toml: 1.88. +[toolchain] +channel = "1.98.1" +components = ["clippy", "rustfmt"] +profile = "minimal" diff --git a/coordinator-rs/schemas/candidate-admission.schema.json b/coordinator-rs/schemas/candidate-admission.schema.json new file mode 100644 index 0000000000..30a2540c77 --- /dev/null +++ b/coordinator-rs/schemas/candidate-admission.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "polis-candidate-admission/1", + "description": "P-026 S1 local admission tuple. Expected digests and operation id must also equal the coordinator request. This is not polis-input/1 or a G01-G16 certificate.", + "type": "object", + "additionalProperties": false, + "required": [ + "candidate_schema", + "engine_version", + "input_digest", + "schedule_digest", + "operation_id" + ], + "properties": { + "candidate_schema": { + "const": "polis-candidate-input/1" + }, + "engine_version": { + "const": "python-conversation/p026-s1" + }, + "input_digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "schedule_digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "operation_id": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } +} diff --git a/coordinator-rs/src/cache.rs b/coordinator-rs/src/cache.rs new file mode 100644 index 0000000000..b5744f74a8 --- /dev/null +++ b/coordinator-rs/src/cache.rs @@ -0,0 +1,71 @@ +//! Bounded warm bundle cache (CO02/CO07 cache-contention stage). +//! +//! It holds the last *coherent published bundle* for a conversation so an +//! unchanged conversation does not re-read the three results tables on every +//! pass. It is not a warm worker/Conversation cache: every changed source is +//! still recomputed from the full authoritative prefix. +//! +//! Two properties matter and are exercised by the R06 stage: +//! 1. lookup and LRU touch are one `&mut self` operation, so an eviction can +//! never interleave between "get" and "move to end" (the Python poller's +//! R06 `KeyError`), and +//! 2. a cached bundle is only usable if its generation still equals the +//! conversation's current `math_tick`, so an update by any other writer +//! turns the entry into a miss instead of a lost update. +use crate::{fault::Fault, store::Bundle}; +use anyhow::Result; +use serde_json::json; + +pub struct WarmCache { + capacity: usize, + /// Least recently used first. + entries: Vec<(i32, Box)>, +} +impl WarmCache { + pub fn new(capacity: usize) -> Self { + Self { + capacity, + entries: Vec::new(), + } + } + pub fn len(&self) -> usize { + self.entries.len() + } + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + pub fn contains(&self, zid: i32) -> bool { + self.entries.iter().any(|(id, _)| *id == zid) + } + /// Remove and return the entry for `zid` when it still matches the current + /// generation. A stale entry is dropped, never returned. + pub fn take(&mut self, zid: i32, current_tick: Option) -> Option> { + let index = self.entries.iter().position(|(id, _)| *id == zid)?; + let (_, bundle) = self.entries.remove(index); + if current_tick == Some(bundle.math_tick) { + Some(bundle) + } else { + tracing::info!(zid, cached = bundle.math_tick, ?current_tick, "stale warm entry discarded"); + None + } + } + /// Insert as most recently used, evicting the least recently used entry. + /// The eviction is announced at the contract's cache-contention stage. + pub fn insert(&mut self, fault: &Fault, zid: i32, bundle: Box) -> Result<()> { + if self.capacity == 0 { + return Ok(()); + } + self.entries.retain(|(id, _)| *id != zid); + while self.entries.len() + 1 > self.capacity { + let victim = self.entries.first().map(|(id, _)| *id); + fault.hit( + "cache_eviction_contends_with_same_zid_update", + &json!({"evicted_zid":victim,"inserted_zid":zid, + "capacity":self.capacity,"cache_len":self.entries.len()}), + )?; + self.entries.remove(0); + } + self.entries.push((zid, bundle)); + Ok(()) + } +} diff --git a/coordinator-rs/src/config.rs b/coordinator-rs/src/config.rs new file mode 100644 index 0000000000..fa7d627d1e --- /dev/null +++ b/coordinator-rs/src/config.rs @@ -0,0 +1,124 @@ +use anyhow::{Result, ensure}; +use std::env; + +#[derive(Clone)] +pub struct Config { + pub database_url: String, + pub math_env: String, + pub python: String, + pub owner: String, + pub storage_agree_value: i64, + pub shard_index: i32, + pub shard_count: i32, + pub allowlist: Vec, + pub page_size: i64, + pub window: i64, + pub lease_seconds: i32, + pub poll_ms: u64, + pub cache_capacity: usize, + /// CO01 fast path: consult the cheap change probe before the authoritative + /// snapshot. The probe never gates a rebuild on its own. + pub incremental: bool, + /// Maximum age of a conversation's authoritative full snapshot. Past it the + /// probe is ignored and the complete reconciliation runs regardless. + pub reconcile_seconds: i32, + /// `off` (default) | `stderr` | `stdout` | a filesystem path. + /// + /// The default is deliberately `off`. A long-running `run` whose stderr is + /// an undrained pipe blocks once the pipe buffer fills, and per-pass metric + /// records fill it an order of magnitude faster than the log lines do; a + /// coordinator must not stall because nobody is reading its telemetry. A + /// deployment chooses its sink explicitly. + pub metrics_sink: String, + /// P-031's `Environment` dimension. Never defaults to `prod`. + pub environment: String, + /// Minimum interval between the bounded backlog/scan-age aggregate. + pub gauge_seconds: u64, + /// Rev6 CO04: the remaining lease a publication must still hold at its + /// final in-transaction authorization, immediately before COMMIT. + pub commit_margin_seconds: f64, +} +fn value(name: &str, default: &str) -> Result +where + T::Err: std::error::Error + Send + Sync + 'static, +{ + Ok(env::var(name) + .unwrap_or_else(|_| default.to_owned()) + .parse()?) +} +impl Config { + pub fn from_env() -> Result { + let c = Self { + database_url: env::var("DATABASE_URL")?, + math_env: env::var("MATH_ENV").unwrap_or_else(|_| "rustproto".into()), + python: env::var("P026_PYTHON").unwrap_or_else(|_| "python3".into()), + owner: uuid::Uuid::new_v4().to_string(), + storage_agree_value: value("STORAGE_AGREE_VALUE", "-1")?, + shard_index: value("POLL_SHARD_INDEX", "0")?, + shard_count: value("POLL_SHARD_COUNT", "1")?, + allowlist: env::var("POLL_ALLOWLIST") + .unwrap_or_default() + .split(',') + .filter(|s| !s.is_empty()) + .map(str::parse) + .collect::>()?, + page_size: value("P026_PAGE_SIZE", "16")?, + window: value("P026_WINDOW", "64")?, + lease_seconds: value("P026_LEASE_SECONDS", "120")?, + poll_ms: value("P026_POLL_MS", "1000")?, + cache_capacity: value("P026_CACHE_CAP", "16")?, + incremental: value::("P026_INCREMENTAL", "1")? != 0, + reconcile_seconds: value("P026_RECONCILE_SECONDS", "3600")?, + metrics_sink: env::var("P026_METRICS").unwrap_or_else(|_| "off".into()), + environment: env::var("P026_ENVIRONMENT").unwrap_or_else(|_| "synthetic".into()), + gauge_seconds: value("P026_GAUGE_SECONDS", "60")?, + commit_margin_seconds: value("P026_COMMIT_MARGIN_SECONDS", "0.5")?, + }; + c.validate()?; + Ok(c) + } + pub fn validate(&self) -> Result<()> { + ensure!( + matches!(self.storage_agree_value, -1 | 1), + "invalid polarity" + ); + ensure!( + self.shard_count > 0 && (0..self.shard_count).contains(&self.shard_index), + "invalid shard" + ); + ensure!((1..=1000).contains(&self.page_size), "invalid page size"); + ensure!(self.window > 0, "invalid trailing window"); + ensure!( + self.lease_seconds > 0 && self.poll_ms > 0, + "invalid interval" + ); + ensure!( + !self.math_env.is_empty() && self.math_env.len() <= 999, + "invalid namespace" + ); + ensure!(self.cache_capacity <= 1024, "invalid warm cache capacity"); + // A non-positive reconciliation ceiling would let the weak hint become + // the only rebuild gate, which Rev5 forbids. + ensure!( + self.reconcile_seconds > 0, + "invalid reconciliation interval" + ); + ensure!(!self.metrics_sink.is_empty(), "invalid metrics sink"); + // A non-positive margin would restore the old "expires_at in the + // future is good enough" gamble Rev6 removed. + ensure!( + self.commit_margin_seconds > 0.0 + && self.commit_margin_seconds < f64::from(self.lease_seconds), + "invalid commit margin" + ); + ensure!( + !self.environment.is_empty() && self.environment.len() <= 64, + "invalid environment dimension" + ); + Ok(()) + } + pub fn accepts(&self, zid: i32) -> bool { + zid % self.shard_count == self.shard_index + && (self.allowlist.is_empty() || self.allowlist.contains(&zid)) + } +} diff --git a/coordinator-rs/src/coordinator.rs b/coordinator-rs/src/coordinator.rs new file mode 100644 index 0000000000..0b147f50e1 --- /dev/null +++ b/coordinator-rs/src/coordinator.rs @@ -0,0 +1,450 @@ +use crate::{ + engine::{self, Source}, + lease::{LeaseState, Renewal}, + metrics::{Tally, count, seconds}, + ordering, + store::{Current, PgStore, Publication, ResultsStore, digest}, +}; +use anyhow::{Result, bail, ensure}; +use postgres::IsolationLevel; +use serde_json::{Value, json}; +use std::time::{Duration, Instant}; + +impl PgStore { + pub fn source(&mut self, zid: i32) -> Result { + let mut tx = self + .client + .build_transaction() + .isolation_level(IsolationLevel::RepeatableRead) + .read_only(true) + .start()?; + // Keyless votes: content order is deterministic, exact duplicate rows remain + // separate events. No ctid or invented primary key, no timestamp cutoff. + // The order is the declared `polis-order/1` normalization, and the agree + // convention reaches it only as that declaration's bound parameter. + ensure!( + ordering::PARAMETER_BINDING == "$2", + "declared ordering parameter is not bound at $2" + ); + let votes_sql = format!("SELECT jsonb_build_object('pid',pid,'tid',tid,'vote',vote,'created',created,'weight_x_32767',weight_x_32767) FROM votes WHERE zid=$1 ORDER BY {} LIMIT 1000001", ordering::order_by()); + let votes: Vec = tx + .query(votes_sql.as_str(), &[&zid, &self.config.storage_agree_value])? + .iter() + .map(|r| r.get(0)) + .collect(); + let comments: Vec = tx.query("SELECT jsonb_build_object('tid',tid,'mod',mod,'is_meta',is_meta,'modified',modified) FROM comments WHERE zid=$1 ORDER BY tid LIMIT 1000001", &[&zid])?.iter().map(|r|r.get(0)).collect(); + let participants: Vec = tx.query("SELECT jsonb_build_object('pid',pid,'mod',mod) FROM participants WHERE zid=$1 ORDER BY pid LIMIT 1000001", &[&zid])?.iter().map(|r|r.get(0)).collect(); + tx.commit()?; + ensure!( + votes.len() <= 1_000_000 + && comments.len() <= 1_000_000 + && participants.len() <= 1_000_000, + "RESOURCE_LIMIT: source exceeds admitted row budget" + ); + let tids = |mod_value: i64| { + comments + .iter() + .filter(|r| r["mod"] == mod_value) + .map(|r| r["tid"].clone()) + .collect::>() + }; + let moderation = json!({"mod_in_tids":tids(1),"mod_out_tids":tids(-1), + "meta_tids":comments.iter().filter(|r|r["is_meta"]==true).map(|r|r["tid"].clone()).collect::>(), + "mod_out_ptpts":participants.iter().filter(|r|r["mod"] == -1).map(|r|r["pid"].clone()).collect::>(), + "lastModTimestamp":Value::Null}); // exact Python poller snapshot profile + let ordering = ordering::manifest( + self.config.storage_agree_value, + ordering::census(&votes), + )?; + // The declared normalization is part of the source identity: changing a + // term or the polarity constant must discard warm state. + let fingerprint = digest(&serde_json::to_vec( + &json!({"votes":votes,"comments":comments,"participants":participants,"ordering":ordering}), + )?); + Ok(Source { + votes, + moderation, + ordering, + fingerprint, + }) + } + pub fn process(&mut self, zid: i32) -> Result { + let started = Instant::now(); + let outcome = self.process_leased(zid); + // CO03: every terminal outcome is counted under its own typed name, so + // "unavailable" can never be read as a crash or as successful work. + match &outcome { + Ok(_) => {} + Err(e) => match LeaseState::of(e) { + Some(LeaseState::Unavailable) => self.tally.lease_unavailable += 1, + Some(LeaseState::Expired) => self.tally.lease_expired += 1, + Some(LeaseState::Fenced) => self.tally.lease_fenced += 1, + None => {} + }, + } + self.metrics.emit( + "conversation", + &[seconds("ConversationLatencySeconds", started.elapsed())], + json!({"zid":zid,"outcome":match &outcome { + Ok(true) => "published".to_owned(), + Ok(false) => "unchanged".to_owned(), + Err(e) => LeaseState::of(e).map_or_else(|| "failed".to_owned(), |s| s.token().to_owned()), + }}), + ); + outcome + } + fn process_leased(&mut self, zid: i32) -> Result { + // No epoch: another owner holds an unexpired lease. Recoverable. + let epoch = self.acquire(zid)?.ok_or(LeaseState::Unavailable)?; + self.tally.lease_acquired += 1; + let mut renewal = Renewal::start(&self.config, zid, epoch)?; + let context = json!({"zid":zid,"math_env":self.config.math_env,"epoch":epoch}); + self.fault.hit("after_lease", &context)?; + let result = self.process_owned(zid, epoch, &renewal); + renewal.stop(); + // Release our own epoch on any decided outcome. The update is + // conditional on (owner_id,owner_epoch), so a transferred or fenced row + // is untouched, epoch history is preserved, and a clean local failure + // does not hold the conversation for the rest of the lease window. + // Only an unclean death leaves a live lease behind, as R05 requires. + if let Err(error) = self.reconnect_if_closed().and_then(|()| self.release(zid, epoch)) { + tracing::warn!(zid, epoch, error=%error, "lease release failed; expiry still bounds it"); + } + result + } + /// Authoritative ownership check at a decision point. A heartbeat loss that + /// the database contradicts is not a loss; anything else stops the work. + fn lease_guard(&mut self, zid: i32, epoch: i64, renewal: &Renewal) -> Result<()> { + if renewal.state().is_none() { + return Ok(()); + } + match self.lease_state(zid, epoch)? { + Some(state) => { + tracing::error!(zid, epoch, state=%state, "lease lost during compute; not publishing"); + Err(state.into()) + } + None => { + tracing::warn!(zid, epoch, "uncertain renewal contradicted by the lease row"); + Ok(()) + } + } + } + fn process_owned(&mut self, zid: i32, epoch: i64, renewal: &Renewal) -> Result { + let expected = self.current_tick(zid)?; + // CO01 incremental discovery. The probe is captured *before* the + // authoritative snapshot it will certify — and carries the database + // time of that observation, so a long compute cannot reset the + // advertised source age. It can only ever *skip* a read; it never + // authorises a rebuild, and it is trusted only while this + // conversation's last full reconciliation is younger than the + // configured ceiling. See src/probe.rs. + let probe = self.probe(zid)?; + self.tally.probed += 1; + let fresh = self.reconciliation(zid)?.is_some_and(|(recorded, age)| { + recorded == probe.value + && age < Duration::from_secs(self.config.reconcile_seconds.max(1) as u64) + }); + let resident = self.cache.take(zid, expected); + // CO06 quiet repair is not negotiable against a fresh hint: an absent + // companion or missing checkpoint provenance must be repaired without + // waiting for future input or for the ceiling, so a metadata-complete + // persisted generation is a precondition of skipping anything. + if self.config.incremental && fresh && self.generation_is_complete(zid)? { + // Rev6: a resident bundle is not evidence about the durable store. + // On the fast path its companions, generation and checkpoint are + // reconciled against the database; a disagreement drops it and + // falls through to the authoritative path below. + let usable = match resident { + Some(bundle) if self.resident_is_intact(zid, &bundle)? => { + self.cache.insert(&self.fault, zid, bundle)?; + true + } + Some(bundle) => { + tracing::warn!( + zid, + math_tick = bundle.math_tick, + "resident bundle contradicted by the store; evicted, repairing" + ); + false + } + // Nothing resident to contradict: the ceiling is what bounds + // this, exactly as it bounds the source hint. + None => true, + }; + if usable { + self.tally.skipped += 1; + tracing::debug!(zid, "source probe unchanged; full snapshot skipped"); + return Ok(false); + } + } + // The authoritative path. Rev7 CO02/CO06: the full-source ceiling must + // *also* independently validate the persisted payloads, so the prior + // generation is re-read and re-hashed from the store here and a + // resident bundle is never the evidence. Payload corruption that leaves + // metadata intact therefore becomes eligible for repair once the ceiling + // elapses, rather than surviving indefinitely behind a warm cache. The + // ceiling is an eligibility threshold, not a measured deadline. + let prior = self.load_current(zid)?; + let read = Instant::now(); + let source = self.source(zid)?; + let source_seconds = read.elapsed(); + self.tally.reconciled += 1; + let context = json!({"zid":zid,"math_env":self.config.math_env,"epoch":epoch,"source_fingerprint":source.fingerprint,"event_count":source.votes.len()}); + self.fault.hit("after_source_selection", &context)?; + // This is an admitted in-memory checkpoint, not durable acknowledgement. + // Source remains authoritative until identical provenance commits with math. + self.fault.hit("after_input_checkpoint", &context)?; + let unchanged = matches!(&prior, + Current::Coherent(b) if b.checkpoint["source_fingerprint"] == source.fingerprint); + if unchanged { + if let Current::Coherent(bundle) = prior { + self.cache.insert(&self.fault, zid, bundle)?; + } + // The authoritative snapshot agreed with the published generation: + // this is the only evidence that lets a later probe skip a read. + self.record_reconciliation(zid, &probe)?; + return Ok(false); + } + let old = match &prior { + Current::Coherent(b) => Some(b.as_ref()), + _ => None, + }; + let guard = || -> Result<()> { + match renewal.state() { + Some(state) => Err(state.into()), + None => Ok(()), + } + }; + let compute_started = Instant::now(); + let computed = engine::compute(&self.config, &self.fault, zid, &source, old, &guard); + let compute_seconds = compute_started.elapsed(); + // Ownership is decided authoritatively here, before any publication, so + // an abort inside compute reports the state the database actually holds. + self.lease_guard(zid, epoch, renewal)?; + let (payloads, checkpoint) = computed?; + let publish_started = Instant::now(); + let mut attempts = 0; + let published = loop { + match self.publish(zid, expected, epoch, checkpoint.clone(), &payloads) { + Ok(result) => break result, + Err(error) => { + let retryable = error + .downcast_ref::() + .and_then(|e| e.code()) + .is_some_and(|code| matches!(code.code(), "40001" | "40P01")); + if !retryable || attempts >= 2 { + return Err(error); + } + attempts += 1; + self.tally.publish_retried += 1; + tracing::warn!(zid, attempts, "retrying whole publication transaction"); + std::thread::sleep(Duration::from_millis(50 * attempts)); + } + } + }; + let publish_seconds = publish_started.elapsed(); + self.metrics.emit( + "reconciliation", + &[ + seconds("SourceReadSeconds", source_seconds), + seconds("ComputeSeconds", compute_seconds), + seconds("PublishSeconds", publish_seconds), + ], + json!({"zid":zid,"events":source.votes.len(),"publication":format!("{published:?}")}), + ); + match published { + Publication::Committed(tick) => { + self.tally.publish_committed += 1; + self.fault.hit("after_ack", &context)?; + // Durable acknowledgement has happened, so the probe captured + // before this snapshot now certifies the published generation. + self.record_reconciliation(zid, &probe)?; + tracing::info!( + zid, + math_env = self.config.math_env, + math_tick = tick, + events = source.votes.len(), + "published" + ); + Ok(true) + } + Publication::Refused(state) => { + self.tally.publish_refused += 1; + Err(state.into()) + } + Publication::Conflict => { + self.tally.publish_conflict += 1; + bail!("publication conflict; source retained") + } + } + } + /// One bounded source pass, instrumented. `SourcePassHealthy` is 1 only + /// when the whole pass completed; a failed pass emits 0 with the same + /// counters, and a dead process emits nothing at all. It is page-loop + /// liveness, deliberately not P-031's A01 `PollHealthy`: a pass in which + /// every conversation failed still completes, and the failure backlog and + /// unrepaired age are what say so. + pub fn cycle(&mut self) -> Result { + let started = Instant::now(); + self.tally = Tally::default(); + let result = self.cycle_pass(); + let mut data = self.tally.data(started.elapsed(), result.is_ok()); + // Bounded aggregate, at most once per gauge interval: CO01 scan age and + // backlog, CO06 oldest unrepaired age. Metadata only, no payload column. + if self + .gauged + .is_none_or(|at| at.elapsed() >= Duration::from_secs(self.config.gauge_seconds)) + { + match self.backlog() { + Ok(backlog) => { + self.gauged = Some(Instant::now()); + data.extend([ + seconds("OldestReconciliationAgeSeconds", backlog.oldest_reconciliation), + count("ReconciliationBacklogConversations", backlog.overdue as f64), + count("FailureBacklogConversations", backlog.failures as f64), + seconds("OldestUnrepairedAgeSeconds", backlog.oldest_unrepaired), + ]); + } + Err(error) => { + // An incomplete observation must never publish health. + tracing::warn!(error=%error, "backlog gauge unavailable this pass"); + } + } + } + data.push(count("MetricsDropped", self.metrics.dropped() as f64)); + self.metrics.emit( + "source_pass", + &data, + json!({"shard":[self.config.shard_index,self.config.shard_count], + "incremental":self.config.incremental, + "reconcile_seconds":self.config.reconcile_seconds}), + ); + result + } + fn cycle_pass(&mut self) -> Result { + let name = format!( + "source-{}-{}", + self.config.shard_index, self.config.shard_count + ); + let mut cursor = self + .client + .query_opt( + "SELECT position FROM coordinator_cursors WHERE math_env=$1 AND consumer=$2", + &[&self.config.math_env, &name], + )? + .and_then(|r| r.get::<_, Value>(0)["zid"].as_i64()) + .unwrap_or(0) as i32; + let rows = self.client.query( + "SELECT zid FROM conversations WHERE zid>$1 ORDER BY zid LIMIT $2", + &[&cursor, &self.config.page_size], + )?; + let mut count = 0; + for r in &rows { + let zid: i32 = r.get(0); + if self.config.accepts(zid) { + self.tally.visited += 1; + let ready = self.client.query_opt("SELECT next_attempt<=clock_timestamp() FROM coordinator_failures WHERE math_env=$1 AND zid=$2", &[&self.config.math_env,&zid])?.is_none_or(|r|r.get::<_,bool>(0)); + if ready { + match self.process(zid) { + Ok(changed) => { + if changed { + count += 1; + self.tally.published += 1; + } + self.client.execute( + "DELETE FROM coordinator_failures WHERE math_env=$1 AND zid=$2", + &[&self.config.math_env, &zid], + )?; + } + // Only a superseded owner stops the daemon. A lease that + // is merely unavailable or elapsed defers one zid and + // must not starve the rest of the sweep. + Err(e) if LeaseState::of(&e).is_some_and(|s| !s.recoverable()) => { + return Err(e); + } + Err(e) => { + match LeaseState::of(&e) { + Some(state) => tracing::warn!(zid,state=%state,"lease deferred; bounded backoff, sweep continues"), + None => tracing::error!(zid,error=%e,"conversation failed; durable retry scheduled"), + } + self.tally.deferred += 1; + self.defer(zid)?; + } + } + } + } + cursor = zid; + } + if rows.len() < self.config.page_size as usize { + cursor = 0; + } + self.client.execute("INSERT INTO coordinator_cursors(math_env,consumer,position) VALUES($1,$2,$3) ON CONFLICT(math_env,consumer) DO UPDATE SET position=excluded.position", &[&self.config.math_env,&name,&json!({"zid":cursor})])?; + tracing::info!( + page_rows = rows.len(), + published = count, + cursor, + "source sweep page completed" + ); + Ok(count) + } + /// Durable bounded backoff for one conversation, shared by failures and by + /// recoverable lease outcomes. + fn defer(&mut self, zid: i32) -> Result<()> { + let mut tx = self.client.transaction()?; + tx.query_one( + "SELECT zid FROM conversations WHERE zid=$1 FOR KEY SHARE", + &[&zid], + )?; + tx.execute("INSERT INTO coordinator_failures(math_env,zid,attempts,next_attempt) VALUES($1,$2,1,clock_timestamp()+interval '1 second') ON CONFLICT(math_env,zid) DO UPDATE SET attempts=LEAST(coordinator_failures.attempts+1,30),next_attempt=clock_timestamp()+make_interval(secs=>LEAST(coordinator_failures.attempts+1,30))", &[&self.config.math_env,&zid])?; + tx.commit()?; + Ok(()) + } + /// One complete bounded-memory pass for --once and the black-box launcher. + /// Strict: any lease refusal ends the pass with its typed exit code. + pub fn once(&mut self) -> Result { + let started = Instant::now(); + self.tally = Tally::default(); + let result = self.once_pass(); + let mut data = self.tally.data(started.elapsed(), result.is_ok()); + match self.backlog() { + Ok(backlog) => data.extend([ + seconds("OldestReconciliationAgeSeconds", backlog.oldest_reconciliation), + count("ReconciliationBacklogConversations", backlog.overdue as f64), + count("FailureBacklogConversations", backlog.failures as f64), + seconds("OldestUnrepairedAgeSeconds", backlog.oldest_unrepaired), + ]), + Err(error) => tracing::warn!(error=%error, "backlog gauge unavailable this pass"), + } + data.push(count("MetricsDropped", self.metrics.dropped() as f64)); + self.metrics.emit( + "source_pass", + &data, + json!({"mode":"once","incremental":self.config.incremental}), + ); + result + } + fn once_pass(&mut self) -> Result { + let mut after = 0; + let mut published = 0; + loop { + let rows = self.client.query( + "SELECT zid FROM conversations WHERE zid>$1 ORDER BY zid LIMIT $2", + &[&after, &self.config.page_size], + )?; + for row in &rows { + let zid: i32 = row.get(0); + if self.config.accepts(zid) { + self.tally.visited += 1; + if self.process(zid)? { + published += 1; + self.tally.published += 1; + } + } + after = zid; + } + if rows.len() < self.config.page_size as usize { + return Ok(published); + } + } + } +} diff --git a/coordinator-rs/src/engine.rs b/coordinator-rs/src/engine.rs new file mode 100644 index 0000000000..d1bde56248 --- /dev/null +++ b/coordinator-rs/src/engine.rs @@ -0,0 +1,397 @@ +//! Sequential framed worker client. Fresh process on every rebuilt source prefix. +use crate::{ + config::Config, + fault::Fault, + store::{Bundle, OriginalPayloads, Payloads, digest}, +}; +use anyhow::{Context, Result, ensure}; +use serde_json::{Value, json}; +use std::{ + fs, + io::{BufRead, BufReader, Write}, + path::Path, + process::{Child, ChildStdin, Command, Stdio}, + sync::mpsc, + time::Duration, +}; + +pub const CANDIDATE_SCHEMA: &str = "polis-candidate-input/1"; +pub const ENGINE_VERSION: &str = "python-conversation/p026-s1"; + +#[derive(Debug, PartialEq)] +pub enum AdmissionError { + Malformed, + Schema, + Engine, + Input, + Schedule, + Operation, + Checksum, +} +impl std::fmt::Display for AdmissionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Malformed => "MALFORMED_CANDIDATE", + Self::Schema => "CANDIDATE_SCHEMA_MISMATCH", + Self::Engine => "ENGINE_VERSION_MISMATCH", + Self::Input => "INPUT_DIGEST_MISMATCH", + Self::Schedule => "SCHEDULE_DIGEST_MISMATCH", + Self::Operation => "OPERATION_ID_MISMATCH", + Self::Checksum => "CHECKSUM_MISMATCH", + }) + } +} +impl std::error::Error for AdmissionError {} +pub fn admit_candidate( + actual: &Value, + requested: &Value, +) -> std::result::Result<(), AdmissionError> { + let keys = [ + "candidate_schema", + "engine_version", + "input_digest", + "schedule_digest", + "operation_id", + ]; + if !actual.as_object().is_some_and(|m| { + m.len() == keys.len() + && keys.iter().all(|k| { + m.get(*k) + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty() && s.len() <= 128) + }) + }) { + return Err(AdmissionError::Malformed); + } + for (key, reason) in keys.into_iter().zip([ + AdmissionError::Schema, + AdmissionError::Engine, + AdmissionError::Input, + AdmissionError::Schedule, + AdmissionError::Operation, + ]) { + if actual[key] != requested[key] { + return Err(reason); + } + } + Ok(()) +} + +/// Exact local candidate envelope; unknown fields require a profile revision. +pub fn admit_checkpoint(checkpoint: &Value, admission: &Value, identity: &Value) -> Result<()> { + let fields = [ + "schema", + "protocol", + "run_id", + "session_id", + "fixture_id", + "checkpoint_id", + "compute_id", + "profile", + "admission", + "output_schema", + "state_schema", + "persistence", + "math_input_cursors", + "observed_state_cursors", + "files", + ]; + ensure!( + checkpoint + .as_object() + .is_some_and(|m| m.len() == fields.len() && fields.iter().all(|k| m.contains_key(*k))), + AdmissionError::Malformed + ); + admit_candidate(&checkpoint["admission"], admission)?; + ensure!( + checkpoint["schema"] == "polis-candidate-checkpoint/1" + && checkpoint["output_schema"] == "polis-candidate-math-output/1" + && checkpoint["protocol"] == "polis-engine/1" + && checkpoint["state_schema"] == "rebuild-prefix/1" + && checkpoint["profile"] == "candidate-profile" + && checkpoint["persistence"] == false, + AdmissionError::Malformed + ); + for key in [ + "fixture_id", + "run_id", + "session_id", + "checkpoint_id", + "compute_id", + ] { + ensure!(checkpoint[key] == identity[key], AdmissionError::Operation); + } + ensure!( + checkpoint["files"].as_object().is_some_and(|m| m.len() == 4 + && ["main", "bidtopid", "ptptstats", "restore"] + .iter() + .all(|k| m.contains_key(*k))), + AdmissionError::Malformed + ); + Ok(()) +} + +pub struct Source { + pub votes: Vec, + pub moderation: Value, + /// The declared `polis-order/1` normalization plus this conversation's + /// `equal_time_census`; shipped verbatim to the worker. + pub ordering: Value, + pub fingerprint: String, +} +fn write_file(root: &Path, name: &str, data: &[u8]) -> Result { + fs::write(root.join(name), data)?; + Ok(json!({"path":name,"bytes":data.len(),"sha256":digest(data)})) +} +fn read_bytes(root: &Path, descriptor: &Value) -> Result> { + ensure!( + descriptor.as_object().is_some_and(|m| m.len() == 3 + && m.contains_key("path") + && m.contains_key("bytes") + && m.contains_key("sha256")), + AdmissionError::Malformed + ); + let name = descriptor["path"] + .as_str() + .context("missing descriptor path")?; + ensure!( + !Path::new(name).is_absolute() + && Path::new(name) + .components() + .all(|c| matches!(c, std::path::Component::Normal(_))), + "unsafe worker output path" + ); + let path = root.join(name); + let mut check = root.to_path_buf(); + for c in Path::new(name).components() { + check.push(c); + ensure!(!check.is_symlink(), "symlink in worker output"); + } + ensure!( + fs::metadata(&path)?.len() <= 256 * 1024 * 1024, + "worker output too large" + ); + let data = fs::read(path)?; + ensure!( + descriptor["bytes"] == data.len() && descriptor["sha256"] == digest(&data), + AdmissionError::Checksum + ); + Ok(data) +} +fn read_file(root: &Path, descriptor: &Value) -> Result { + crate::wire::parse(&read_bytes(root, descriptor)?).map_err(|_| AdmissionError::Malformed.into()) +} +struct Worker { + child: Child, + input: ChildStdin, + responses: mpsc::Receiver>, + request: i64, + run: String, + session: String, +} +impl Drop for Worker { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} +impl Worker { + fn start(c: &Config, input: &Path, output: &Path) -> Result { + let mut child = Command::new(&c.python) + .args(["-m", "polismath.engine_adapter", "--input-root"]) + .arg(input) + .arg("--output-root") + .arg(output) + .env("OMP_NUM_THREADS", "1") + .env("OPENBLAS_NUM_THREADS", "1") + .env("MKL_NUM_THREADS", "1") + .env("PYTHONDONTWRITEBYTECODE", "1") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn()?; + let stdin = child.stdin.take().context("worker stdin missing")?; + let stdout = child.stdout.take().context("worker stdout missing")?; + let (sender, responses) = mpsc::sync_channel(1); + std::thread::spawn(move || { + let mut reader = BufReader::new(stdout); + loop { + let mut line = Vec::new(); + let result = (|| -> Result { + use std::io::Read; + let count = reader.by_ref().take(65537).read_until(b'\n', &mut line)?; + ensure!( + count > 0 && count <= 65536 && line.last() == Some(&b'\n'), + "malformed/EOF worker response" + ); + Ok(crate::wire::parse(&line)?) + })(); + let stop = result.is_err(); + if sender.send(result).is_err() || stop { + break; + } + } + }); + Ok(Self { + child, + input: stdin, + responses, + request: 0, + run: uuid::Uuid::new_v4().to_string(), + session: uuid::Uuid::new_v4().to_string(), + }) + } + fn call(&mut self, op: &str, payload: Value, guard: &dyn Fn() -> Result<()>) -> Result { + self.request += 1; + let req = json!({"protocol":"polis-engine/1","run_id":self.run,"session_id":self.session,"request_id":self.request,"op":op,"payload":payload}); + let bytes = serde_json::to_vec(&req)?; + ensure!(bytes.len() < 65536, "control request too large"); + self.input.write_all(&bytes)?; + self.input.write_all(b"\n")?; + self.input.flush()?; + // Poll so a lease lost mid-compute aborts the operation instead of + // waiting out the whole worker timeout. + let deadline = std::time::Instant::now() + Duration::from_secs(120); + let response = loop { + guard()?; + match self.responses.recv_timeout(Duration::from_millis(200)) { + Ok(response) => break response?, + Err(mpsc::RecvTimeoutError::Timeout) => { + ensure!(std::time::Instant::now() < deadline, "worker timeout/exit"); + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + return Err(anyhow::anyhow!("worker timeout/exit")); + } + } + }; + ensure!( + response.as_object().is_some_and(|m| m.len() == 6 + && m.keys().all(|k| matches!( + k.as_str(), + "protocol" | "run_id" | "session_id" | "request_id" | "ok" | "result" | "error" + ))), + "invalid response envelope" + ); + ensure!( + response["protocol"] == "polis-engine/1" + && response["run_id"] == self.run + && response["session_id"] == self.session + && response["request_id"] == self.request, + "worker identity mismatch" + ); + ensure!( + response["ok"] == true, + "worker rejected operation: {}", + response["error"] + ); + Ok(response["result"].clone()) + } +} +pub fn compute( + c: &Config, + fault: &Fault, + zid: i32, + source: &Source, + prior: Option<&Bundle>, + guard: &dyn Fn() -> Result<()>, +) -> Result<(Payloads, Value)> { + let input = tempfile::tempdir()?; + let output = tempfile::tempdir()?; + let mut vote_bytes = Vec::new(); + for (i, v) in source.votes.iter().enumerate() { + let row = json!({"slot":i+1,"source_ordinal":i,"stream_ordinal":i,"created_ms":v["created"],"pid":v["pid"],"tid":v["tid"],"raw_vote":v["vote"],"weight_x_32767":v["weight_x_32767"]}); + serde_json::to_writer(&mut vote_bytes, &row)?; + vote_bytes.push(b'\n'); + } + let votes = write_file(input.path(), "votes.jsonl", &vote_bytes)?; + let mut mod_bytes = serde_json::to_vec(&json!({"slot":1,"state":source.moderation}))?; + mod_bytes.push(b'\n'); + let mods = write_file(input.path(), "moderation.jsonl", &mod_bytes)?; + let parent = prior.map(|b| json!({"profile":"rebuild-prefix/1","source_fingerprint":b.checkpoint["source_fingerprint"],"math_tick":b.math_tick,"payload_digests":b.checkpoint["payload_digests"]})); + // `ordering` is the declared contract term the coordinator itself ordered by. + let manifest = json!({"schema":CANDIDATE_SCHEMA,"fixture_id":zid,"storage_agree_value":c.storage_agree_value, + "ordering":source.ordering,"votes":votes,"moderation":mods,"parent":parent}); + let manifest_desc = write_file(input.path(), "input.json", &serde_json::to_vec(&manifest)?)?; + let mut ops = Vec::new(); + if let Some(p) = prior { + let desc = write_file( + input.path(), + "restore.json", + &serde_json::to_vec(&p.payloads.main)?, + )?; + ops.push(json!({"op":"restore","payload":{"profile":"rebuild-prefix/1","parent":parent,"payload":desc,"vote_cursor":source.votes.len(),"moderation_cursor":1}})); + } else { + if !source.votes.is_empty() { + ops.push(json!({"op":"apply_votes","payload":{"from_slot_exclusive":0,"to_slot_inclusive":source.votes.len()}})); + } + ops.push(json!({"op":"apply_moderation","payload":{"from_slot_exclusive":0,"to_slot_inclusive":1}})); + } + ops.extend([ + json!({"op":"compute","payload":{"compute_id":"compute-0","logical_clock":0}}), + json!({"op":"snapshot","payload":{"checkpoint_id":"checkpoint-0"}}), + json!({"op":"close","payload":{}}), + ]); + let schedule_desc = write_file( + input.path(), + "schedule.json", + &serde_json::to_vec(&json!({"schema":"polis-schedule/1","operations":ops}))?, + )?; + let mut worker = Worker::start(c, input.path(), output.path())?; + let operation_id = uuid::Uuid::new_v4().to_string(); + let admission = json!({"candidate_schema":CANDIDATE_SCHEMA,"engine_version":ENGINE_VERSION, + "input_digest":manifest_desc["sha256"],"schedule_digest":schedule_desc["sha256"],"operation_id":operation_id}); + let initialized = worker.call("initialize",json!({"input_manifest":manifest_desc,"resolved_schedule":schedule_desc, + "admission":admission,"required_capabilities":["rebuild-prefix/1","snapshot-moderation/1"], + "config":{"profile":"candidate-profile","seed":42,"pca_mode":"powerit","empty_contract":true,"init_vector":"engine-default"}}),guard)?; + admit_candidate(&initialized["admission"], &admission)?; + let context = json!({"operation_id":operation_id,"zid":zid,"math_env":c.math_env,"fingerprint":source.fingerprint,"worker_pid":worker.child.id()}); + fault.hit("before_worker_apply", &context)?; + let mut checkpoint = Value::Null; + for op in ops { + let name = op["op"].as_str().context("operation name")?; + if name == "restore" { + fault.hit("before_restore", &context)?; + } + let result = worker.call(name, op["payload"].clone(), guard)?; + if name == "restore" { + fault.hit("after_restore", &context)?; + } + if name == "compute" { + fault.hit("after_worker_compute", &context)?; + } + if name == "snapshot" { + checkpoint = read_file(output.path(), &result["manifest"])?; + } + } + ensure!(worker.child.wait()?.success(), "nonzero worker exit"); + admit_checkpoint( + &checkpoint, + &admission, + &json!({ + "fixture_id":zid,"run_id":worker.run,"session_id":worker.session, + "checkpoint_id":"checkpoint-0","compute_id":"compute-0"}), + )?; + let cursors = json!({"votes":{"slot":source.votes.len(),"sha256":digest(&vote_bytes)},"moderation":{"slot":1,"sha256":digest(&mod_bytes)}}); + guard()?; + ensure!( + checkpoint["observed_state_cursors"] == cursors + && checkpoint["math_input_cursors"] == cursors, + "worker prefix mismatch" + ); + let files = &checkpoint["files"]; + let payloads = Payloads::from_originals(OriginalPayloads { + main: read_bytes(output.path(), &files["main"])?, + bidtopid: read_bytes(output.path(), &files["bidtopid"])?, + ptptstats: read_bytes(output.path(), &files["ptptstats"])?, + }) + .map_err(|_| AdmissionError::Malformed)?; + payloads + .validate(zid) + .map_err(|_| AdmissionError::Malformed)?; + Ok(( + payloads, + json!({"schema":"polis-coordinator/1","operation_id":operation_id,"admission":admission,"source_fingerprint":source.fingerprint, + "profile":"candidate-profile","lifecycle":"rebuild-prefix/1","cursors":cursors,"event_count":source.votes.len(), + "storage_agree_value":c.storage_agree_value,"ordering":source.ordering}), + )) +} diff --git a/coordinator-rs/src/fault.rs b/coordinator-rs/src/fault.rs new file mode 100644 index 0000000000..f003dbcf1c --- /dev/null +++ b/coordinator-rs/src/fault.rs @@ -0,0 +1,107 @@ +//! Process-external file latches; entirely absent from release control flow. +use anyhow::{Result, ensure}; +use postgres::Client; + +pub const STAGES: &[&str] = &[ + "after_source_selection", + "after_input_checkpoint", + "before_worker_apply", + "after_worker_compute", + "after_lease", + "before_ticks", + "after_ticks", + "before_bidtopid", + "after_bidtopid", + "before_ptptstats", + "after_ptptstats", + "before_main", + "after_main", + "before_commit", + "after_commit", + "after_ack", + "before_restore", + "after_restore", + "before_cursor", + "after_cursor", + "before_sweep", + "after_sweep", + "bundle_pinned", + "before_companion_join", + "cache_eviction_contends_with_same_zid_update", +]; +#[derive(Default)] +pub struct Fault { + #[cfg(feature = "fault-injection")] + directory: Option, +} +impl Fault { + pub fn reject_release_control() -> Result<()> { + #[cfg(not(feature = "fault-injection"))] + ensure!( + std::env::var_os("P026_FAULT_DIR").is_none(), + "FAULT-CONTROL-REFUSED" + ); + Ok(()) + } + pub fn new(client: &mut Client, namespace: &str) -> Result { + Self::reject_release_control()?; + #[cfg(feature = "fault-injection")] + { + let directory = std::env::var_os("P026_FAULT_DIR").map(std::path::PathBuf::from); + if directory.is_some() { + ensure!( + !matches!(namespace, "prod" | "preprod" | "dev"), + "test namespace refused" + ); + let marker: bool = client + .query_one( + "SELECT EXISTS(SELECT 1 FROM p026_test_marker WHERE namespace=$1)", + &[&namespace], + )? + .get(0); + ensure!(marker, "synthetic database/namespace marker required"); + } + Ok(Self { directory }) + } + #[cfg(not(feature = "fault-injection"))] + { + let _ = (client, namespace); + Ok(Self {}) + } + } + pub fn hit(&self, stage: &str, context: &serde_json::Value) -> Result<()> { + #[cfg(feature = "fault-injection")] + if let Some(dir) = &self.directory { + use std::{ + fs, thread, + time::{Duration, Instant}, + }; + let arm = dir.join("arm.json"); + if arm.exists() { + let request: serde_json::Value = serde_json::from_slice(&fs::read(&arm)?)?; + if request["stage"] == stage { + ensure!( + request["protocol"] == "polis-fault-control/1", + "invalid fault protocol" + ); + let ack = serde_json::json!({"protocol":"polis-fault-control/1", "stage":stage, + "run_id":request["run_id"], "operation_id":request["operation_id"], + "pid":std::process::id(), "state":"reached-and-blocked", "context":context}); + fs::write(dir.join("ack.tmp"), serde_json::to_vec(&ack)?)?; + fs::rename(dir.join("ack.tmp"), dir.join("ack.json"))?; + let start = Instant::now(); + while !dir.join("release").exists() { + ensure!( + start.elapsed() < Duration::from_secs(180), + "fault latch timeout" + ); + thread::sleep(Duration::from_millis(10)); + } + fs::remove_file(arm)?; + } + } + } + let _ = (stage, context); + Ok(()) + } +} diff --git a/coordinator-rs/src/lease.rs b/coordinator-rs/src/lease.rs new file mode 100644 index 0000000000..d5fa5bdfe4 --- /dev/null +++ b/coordinator-rs/src/lease.rs @@ -0,0 +1,168 @@ +//! CO03 lease state and heartbeat renewal. +//! +//! Rev5 requires three distinguishable outcomes instead of one refusal, and +//! renewal during compute that can never resurrect an expired or transferred +//! epoch. `Unavailable` and `Expired` are recoverable: the daemon defers the +//! conversation and keeps working. `Fenced` means this owner's publication +//! authority is gone for good, so the process stops instead of retrying. +use crate::config::Config; +use anyhow::Result; +use postgres::{Client, NoTls}; +use std::{ + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + }, + thread::{self, JoinHandle}, + time::{Duration, Instant}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LeaseState { + /// Another owner holds an unexpired lease on this (math_env,zid). + Unavailable, + /// The lease row is still ours, but our own lease elapsed in database time. + Expired, + /// Owner or epoch superseded; this owner may never publish this generation. + Fenced, +} +impl LeaseState { + pub fn token(self) -> &'static str { + match self { + Self::Unavailable => "LEASE-UNAVAILABLE", + Self::Expired => "LEASE-EXPIRED", + Self::Fenced => "FENCED", + } + } + /// Distinct process exit codes; see README "Ownership outcomes". + pub fn exit_code(self) -> i32 { + match self { + Self::Fenced => 3, + Self::Unavailable => 4, + Self::Expired => 5, + } + } + /// Recoverable outcomes defer one conversation; they never kill the daemon. + pub fn recoverable(self) -> bool { + !matches!(self, Self::Fenced) + } + /// The typed outcome carried by an error, if any. + pub fn of(error: &anyhow::Error) -> Option { + error.downcast_ref::().copied() + } +} +impl std::fmt::Display for LeaseState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.token()) + } +} +impl std::error::Error for LeaseState {} + +/// Conditional renewal. Never extends a lease that is not currently ours and +/// unexpired, so an expired or transferred epoch cannot be resurrected. +fn renew(client: &mut Client, c: &Config, zid: i32, epoch: i64) -> Result> { + let mut tx = client.transaction()?; + tx.query_one( + "SELECT zid FROM conversations WHERE zid=$1 FOR KEY SHARE", + &[&zid], + )?; + let renewed = tx.execute("UPDATE coordinator_leases SET expires_at=clock_timestamp()+make_interval(secs=>$5::int) WHERE math_env=$1 AND zid=$2 AND owner_id=$3 AND owner_epoch=$4 AND expires_at>clock_timestamp()", &[&c.math_env,&zid,&c.owner,&epoch,&c.lease_seconds])?; + if renewed == 1 { + tx.commit()?; + return Ok(None); + } + let observed = tx.query_opt("SELECT owner_id,owner_epoch,expires_at>clock_timestamp() FROM coordinator_leases WHERE math_env=$1 AND zid=$2", &[&c.math_env,&zid])?; + tx.commit()?; + match classify(observed.as_ref(), c, epoch) { + Some(state) => Ok(Some(state)), + // Still ours and unexpired, yet the identical conditional update matched + // nothing: uncertain, and an uncertain renewal is not ownership proof. + None => anyhow::bail!("uncertain lease renewal"), + } +} + +/// Classify an observed lease row for this owner and epoch. +/// `None` means the row is still ours and unexpired. +pub fn classify(row: Option<&postgres::Row>, c: &Config, epoch: i64) -> Option { + let Some(r) = row else { + return Some(LeaseState::Fenced); + }; + if r.get::<_, String>(0) != c.owner || r.get::<_, i64>(1) != epoch { + Some(LeaseState::Fenced) + } else if r.get::<_, bool>(2) { + None + } else { + Some(LeaseState::Expired) + } +} + +/// Heartbeat on the lease row, on its own connection, for the life of one +/// compute. The publication transaction is never held open across it. +pub struct Renewal { + lost: Arc>>, + stop: Arc, + handle: Option>, +} +impl Renewal { + pub fn start(config: &Config, zid: i32, epoch: i64) -> Result { + let mut client = Client::connect(&config.database_url, NoTls)?; + client.batch_execute("SET statement_timeout='5s'; SET lock_timeout='1s'; SET application_name='p026-lease-renewal'")?; + let lost: Arc>> = Arc::new(Mutex::new(None)); + let stop = Arc::new(AtomicBool::new(false)); + let (config, thread_lost, thread_stop) = (config.clone(), lost.clone(), stop.clone()); + let handle = thread::spawn(move || { + let lease = Duration::from_secs(config.lease_seconds.max(1) as u64); + // Renew well inside the window so one lost round trip is survivable. + let interval = (lease / 3).max(Duration::from_millis(50)); + let mut confirmed = Instant::now(); + let set = |state: LeaseState| { + if let Ok(mut slot) = thread_lost.lock() { + *slot = Some(state); + } + }; + while !thread_stop.load(Ordering::SeqCst) { + let wake = Instant::now() + interval; + while Instant::now() < wake { + if thread_stop.load(Ordering::SeqCst) { + return; + } + thread::sleep(Duration::from_millis(10).min(wake.saturating_duration_since(Instant::now()))); + } + match renew(&mut client, &config, zid, epoch) { + Ok(None) => confirmed = Instant::now(), + Ok(Some(state)) => { + set(state); + return; + } + // An uncertain renewal is not ownership proof: once a full + // lease has passed without a confirmed renewal, stop. + Err(_) if confirmed.elapsed() >= lease => { + set(LeaseState::Expired); + return; + } + Err(_) => {} + } + } + }); + Ok(Self { + lost, + stop, + handle: Some(handle), + }) + } + /// The last definite loss observed by the heartbeat, if any. + pub fn state(&self) -> Option { + self.lost.lock().ok().and_then(|slot| *slot) + } + pub fn stop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} +impl Drop for Renewal { + fn drop(&mut self) { + self.stop(); + } +} diff --git a/coordinator-rs/src/lib.rs b/coordinator-rs/src/lib.rs new file mode 100644 index 0000000000..fec882cbb1 --- /dev/null +++ b/coordinator-rs/src/lib.rs @@ -0,0 +1,12 @@ +pub mod cache; +pub mod config; +pub mod coordinator; +pub mod engine; +pub mod fault; +pub mod lease; +pub mod metrics; +pub mod ordering; +pub mod probe; +pub mod reader; +pub mod store; +pub mod wire; diff --git a/coordinator-rs/src/main.rs b/coordinator-rs/src/main.rs new file mode 100644 index 0000000000..2f3023bda1 --- /dev/null +++ b/coordinator-rs/src/main.rs @@ -0,0 +1,173 @@ +use anyhow::{Result, bail}; +use polis_coordinator::{ + config::Config, + fault::Fault, + lease::LeaseState, + store::{PgStore, ResultsStore}, +}; +use serde_json::json; + +fn run() -> Result<()> { + Fault::reject_release_control()?; + let mode = std::env::args().nth(1).unwrap_or_else(|| "run".into()); + if mode == "stages" { + println!("{}", json!(polis_coordinator::fault::STAGES)); + return Ok(()); + } + if mode == "metrics" { + // The declared CO01/CO06 metric catalog, generated from the code that + // emits it, so the P-031 evidence file cannot drift from reality. + println!("{}", polis_coordinator::metrics::catalog_json()); + return Ok(()); + } + let config = Config::from_env()?; + let mut store = PgStore::connect(config)?; + match mode.as_str() { + "migrate" => store.migrate(), + "once" => { + let n = store.once()?; + println!("{}", json!({"status":"DONE","published":n})); + Ok(()) + } + #[cfg(feature = "fault-injection")] + "publish-fixture" => { + let marker: bool = store + .client + .query_one( + "SELECT EXISTS(SELECT 1 FROM p026_test_marker WHERE namespace=$1)", + &[&store.config.math_env], + )? + .get(0); + anyhow::ensure!(marker, "test database required"); + let path = std::env::args() + .nth(2) + .ok_or_else(|| anyhow::anyhow!("fixture path required"))?; + let v: serde_json::Value = serde_json::from_slice(&std::fs::read(path)?)?; + let zid: i32 = serde_json::from_value(v["zid"].clone())?; + // Fixture ingress has no worker files. Preserve its explicitly supplied + // bytes, or serialize the fixture values once as the fixture origin. + // Synthesized bytes are NOT engine originals: they are re-encoded from + // the fixture's parsed values, so `validate_originals()` is tautological + // for those rows and proves nothing about worker output custody. The + // provenance is announced on stderr so a fixture row is never mistaken + // for an admitted worker byte stream. + let mut synthesized = Vec::new(); + let mut raw = |key: &str| -> Result> { + if let Some(bytes) = v["payloads"]["originals"].get(key) { + Ok(serde_json::from_value(bytes.clone())?) + } else { + synthesized.push(key.to_string()); + Ok(serde_json::to_vec(&v["payloads"][key])?) + } + }; + let originals = polis_coordinator::store::OriginalPayloads { + main: raw("main")?, + bidtopid: raw("bidtopid")?, + ptptstats: raw("ptptstats")?, + }; + eprintln!( + "originals: synthesized={synthesized:?} (re-encoded fixture values, not engine originals)" + ); + let payloads = polis_coordinator::store::Payloads::from_originals(originals)?; + let mut checkpoint = v["checkpoint"].clone(); + if checkpoint.get("operation_id").is_none() { + checkpoint["operation_id"] = json!(uuid::Uuid::new_v4().to_string()); + } + let epoch = store.acquire(zid)?.ok_or(LeaseState::Unavailable)?; + let expected = v["expected_tick"].as_i64(); + let result = store.publish(zid, expected, epoch, checkpoint, &payloads)?; + store.release(zid, epoch)?; + println!("{result:?}"); + // A refusal is not a successful one-shot run. `publish-fixture` is + // a strict command like `once`, so an ownership refusal goes + // through the same typed exit mapper instead of being printed and + // exiting 0. Conflict keeps its distinct handling: a stale expected + // tick is a normal outcome that overwrote nothing, and the caller + // reads it from the printed result. + match result { + polis_coordinator::store::Publication::Refused(state) => Err(state.into()), + polis_coordinator::store::Publication::Conflict + | polis_coordinator::store::Publication::Committed(_) => Ok(()), + } + } + "reader" => { + let consumer = std::env::args().nth(2).unwrap_or_else(|| "cli".into()); + let bundles = store.reader_page(&consumer)?; + println!("{}", serde_json::to_string(&bundles)?); + Ok(()) + } + "read" => { + let zid: i32 = std::env::args() + .nth(2) + .ok_or_else(|| anyhow::anyhow!("zid required"))? + .parse()?; + if let polis_coordinator::store::Current::Coherent(b) = store.load_current(zid)? { + store + .fault + .hit("bundle_pinned", &json!({"zid":zid,"tick":b.math_tick}))?; + store.fault.hit( + "before_companion_join", + &json!({"zid":zid,"tick":b.math_tick}), + )?; + println!("{}", serde_json::to_string(&b)?); + Ok(()) + } else { + bail!("absent/inconsistent bundle") + } + } + "scan" => { + let after: i32 = std::env::args() + .nth(2) + .unwrap_or_else(|| "0".into()) + .parse()?; + store.fault.hit("before_sweep", &json!({"after":after}))?; + let rows = store.scan_current(after, store.config.page_size)?; + store.fault.hit("after_sweep", &json!({"rows":rows}))?; + println!("{}", serde_json::to_string(&rows)?); + Ok(()) + } + "poll" => { + let high: i64 = std::env::args() + .nth(2) + .unwrap_or_else(|| "0".into()) + .parse()?; + store.fault.hit("before_cursor", &json!({"high":high}))?; + let rows = + store.poll_changes(high, store.config.window, (-1, 0), store.config.page_size)?; + store.fault.hit("after_cursor", &json!({"rows":rows}))?; + println!("{}", serde_json::to_string(&rows)?); + Ok(()) + } + "run" => loop { + if let Err(e) = store.cycle() { + // Only a superseded owner ends the daemon. + if LeaseState::of(&e).is_some_and(|s| !s.recoverable()) { + return Err(e); + } + tracing::error!(error=%e,"cycle failed; cursor retained for retry"); + if store.client.is_closed() + && let Ok(reconnected) = PgStore::connect(store.config.clone()) + { + store = reconnected; + } + } + std::thread::sleep(std::time::Duration::from_millis(store.config.poll_ms)); + }, + _ => bail!("unknown command"), + } +} +fn main() { + tracing_subscriber::fmt() + .json() + .with_writer(std::io::stderr) + .init(); + if let Err(e) = run() { + let code = LeaseState::of(&e).map_or(1, LeaseState::exit_code); + let sqlstate = e + .downcast_ref::() + .and_then(|e| e.code()) + .map(|c| c.code()); + tracing::error!(error=%e,exit_code=code,sqlstate,"coordinator failed"); + std::process::exit(code); + } +} diff --git a/coordinator-rs/src/metrics.rs b/coordinator-rs/src/metrics.rs new file mode 100644 index 0000000000..9a9defc663 --- /dev/null +++ b/coordinator-rs/src/metrics.rs @@ -0,0 +1,354 @@ +//! CO01/CO06 observability: the counters the contract names, in a form the +//! P-031 alarm catalog can consume. +//! +//! The contract requires the source pass to "expose scan age, backlog and +//! failures" (CO01), "oldest unrepaired age" (CO06) and "maximum pass age" +//! (CO05). None of that existed. This module emits those, plus per-conversation +//! latency and the typed lease/publication outcome counters, as **CloudWatch +//! Embedded Metric Format** records in namespace `Polis/Math` with P-031's two +//! fixed dimensions `Environment` and `MathEnv` — and no per-conversation, +//! per-run or per-instance dimension, which P-031 forbids. +//! +//! There is no AWS client here and this crate takes no AWS dependency. A +//! [`Sink`] receives finished records; the prototype ships a JSON-lines sink +//! (stderr, stdout or a file) and a null sink. In a deployment the same records +//! reach CloudWatch through the awslogs path P-031 already describes; nothing +//! in this crate publishes a metric itself. +//! +//! Emission is best effort and never fails an operation: a coordinator that +//! cannot write a metric line still must not lose a vote. Failures are logged +//! once per record and counted. +use crate::config::Config; +use anyhow::Result; +use serde_json::{Value, json}; +use std::{ + fs::OpenOptions, + io::Write, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +/// P-031 rev2 namespace for every math signal. +pub const NAMESPACE: &str = "Polis/Math"; +/// The record shape; pinned so a log metric filter can match on it. +pub const SCHEMA: &str = "polis-metrics/1"; + +/// CloudWatch units, spelled exactly as the API spells them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Unit { + Count, + Seconds, +} +impl Unit { + pub fn as_str(self) -> &'static str { + match self { + Self::Count => "Count", + Self::Seconds => "Seconds", + } + } +} + +/// One metric value in one record. +#[derive(Debug, Clone)] +pub struct Datum { + pub name: &'static str, + pub value: f64, + pub unit: Unit, +} +pub fn count(name: &'static str, value: impl Into) -> Datum { + Datum { + name, + value: value.into(), + unit: Unit::Count, + } +} +pub fn seconds(name: &'static str, value: Duration) -> Datum { + Datum { + name, + value: value.as_secs_f64(), + unit: Unit::Seconds, + } +} + +/// Every metric this crate can emit, with the P-031 alarm it feeds. +/// `audit_stages.py` and `evidence/metrics-catalog.json` are generated from it, +/// so a metric cannot be added or dropped without the catalog noticing. +pub struct Declared { + pub name: &'static str, + pub unit: Unit, + /// The CloudWatch statistic the alarm should use. + pub statistic: &'static str, + /// P-031 catalog row this metric actually satisfies, or `""`. Rev7's + /// observability admission: none of these *is* A01 `PollHealthy`, + /// A02 `PublishLagSeconds` or A03 `ObserverHealthy`, so no row claims one. + pub alarm: &'static str, + pub meaning: &'static str, +} +pub const CATALOG: &[Declared] = &[ + Declared { name: "SourcePassHealthy", unit: Unit::Count, statistic: "Minimum", alarm: "", + meaning: "1 only when a whole source pass completed; 0 on a failed pass, missing when the process is gone. This is page-loop liveness. It is NOT P-031 A01 PollHealthy, which requires both the vote and the moderation poll to have succeeded recently: a pass in which every conversation failed still completes" }, + Declared { name: "SourcePassSeconds", unit: Unit::Seconds, statistic: "Maximum", alarm: "", + meaning: "wall time of one bounded source pass" }, + Declared { name: "SourcePassConversations", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "conversations visited by the pass, before shard/backoff filtering" }, + Declared { name: "SourcePassProbed", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "conversations whose cheap change probe was evaluated" }, + Declared { name: "SourcePassSkipped", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "conversations the probe found unchanged, so no full source read was taken" }, + Declared { name: "SourcePassReconciled", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "conversations that took the authoritative full snapshot this pass" }, + Declared { name: "SourcePassPublished", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "conversations that committed a new generation this pass" }, + Declared { name: "SourcePassDeferred", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "conversations deferred this pass with durable bounded backoff" }, + Declared { name: "OldestReconciliationAgeSeconds", unit: Unit::Seconds, statistic: "Maximum", alarm: "", + meaning: "CO01 scan age: age of the oldest authoritative full source snapshot for this shard's candidates, measured from before the source read. The incremental fast path is only sound while this is bounded. It is NOT P-031 A02 PublishLagSeconds: a quiet conversation's source age grows toward the ceiling with no pending math at all" }, + Declared { name: "ReconciliationBacklogConversations", unit: Unit::Count, statistic: "Maximum", alarm: "", + meaning: "CO01 backlog: candidate conversations never reconciled, or overdue for reconciliation" }, + Declared { name: "FailureBacklogConversations", unit: Unit::Count, statistic: "Maximum", alarm: "", + meaning: "CO01 failures: candidate conversations currently in durable backoff; a conversation this shard/allowlist would never attempt is not counted" }, + Declared { name: "OldestUnrepairedAgeSeconds", unit: Unit::Seconds, statistic: "Maximum", alarm: "", + meaning: "CO06 oldest unrepaired age: how long the oldest still-failing candidate conversation has been failing. Scoped to this shard and allowlist, like every other gauge here. Not P-031 A02" }, + Declared { name: "ConversationLatencySeconds", unit: Unit::Seconds, statistic: "Maximum", alarm: "", + meaning: "per-zid probe -> source -> compute -> publish wall time" }, + Declared { name: "SourceReadSeconds", unit: Unit::Seconds, statistic: "Maximum", alarm: "", + meaning: "per-zid authoritative snapshot read time" }, + Declared { name: "ComputeSeconds", unit: Unit::Seconds, statistic: "Maximum", alarm: "", + meaning: "per-zid worker compute time, worker process lifetime included" }, + Declared { name: "PublishSeconds", unit: Unit::Seconds, statistic: "Maximum", alarm: "", + meaning: "per-zid publication transaction time, retries included" }, + Declared { name: "LeaseAcquired", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "CO03 lease outcome: this owner took or renewed ownership at an epoch" }, + Declared { name: "LeaseUnavailable", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "CO03 lease outcome: another owner holds an unexpired lease" }, + Declared { name: "LeaseExpired", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "CO03 lease outcome: this owner's own lease elapsed in database time" }, + Declared { name: "LeaseFenced", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "CO03 lease outcome: owner or epoch superseded; terminal for this owner" }, + Declared { name: "PublishCommitted", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "CO04 publication outcome: one generation committed" }, + Declared { name: "PublishConflict", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "CO04 publication outcome: stale expected tick, nothing overwritten" }, + Declared { name: "PublishRefused", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "CO04 publication outcome: ownership refused inside the transaction" }, + Declared { name: "PublishRetried", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "CO04 publication outcome: whole-transaction retry after 40001/40P01" }, + Declared { name: "PublishUncertain", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "CO04 publication outcome: lost COMMIT response resolved by checkpoint identity" }, + Declared { name: "MetricsDropped", unit: Unit::Count, statistic: "Sum", alarm: "", + meaning: "records this process failed to write; a nonzero value means the other series are incomplete" }, +]; + +/// Per-pass counters. Reset at the start of every source pass and emitted as +/// one record when the pass ends, healthy or not. +#[derive(Debug, Default, Clone, Copy)] +pub struct Tally { + pub visited: u32, + pub probed: u32, + pub skipped: u32, + pub reconciled: u32, + pub published: u32, + pub deferred: u32, + pub lease_acquired: u32, + pub lease_unavailable: u32, + pub lease_expired: u32, + pub lease_fenced: u32, + pub publish_committed: u32, + pub publish_conflict: u32, + pub publish_refused: u32, + pub publish_retried: u32, + pub publish_uncertain: u32, +} +impl Tally { + pub fn data(&self, elapsed: Duration, healthy: bool) -> Vec { + vec![ + count("SourcePassHealthy", u32::from(healthy)), + seconds("SourcePassSeconds", elapsed), + count("SourcePassConversations", self.visited), + count("SourcePassProbed", self.probed), + count("SourcePassSkipped", self.skipped), + count("SourcePassReconciled", self.reconciled), + count("SourcePassPublished", self.published), + count("SourcePassDeferred", self.deferred), + count("LeaseAcquired", self.lease_acquired), + count("LeaseUnavailable", self.lease_unavailable), + count("LeaseExpired", self.lease_expired), + count("LeaseFenced", self.lease_fenced), + count("PublishCommitted", self.publish_committed), + count("PublishConflict", self.publish_conflict), + count("PublishRefused", self.publish_refused), + count("PublishRetried", self.publish_retried), + count("PublishUncertain", self.publish_uncertain), + ] + } +} + +/// Where finished records go. Deliberately not an AWS client. +pub trait Sink: Send { + fn write(&mut self, record: &Value) -> Result<()>; + /// Human name for logs and evidence. + fn describe(&self) -> String; +} + +/// Discards everything. Used by `--metrics off` and by unit tests. +pub struct NullSink; +impl Sink for NullSink { + fn write(&mut self, _record: &Value) -> Result<()> { + Ok(()) + } + fn describe(&self) -> String { + "null".into() + } +} + +/// One JSON object per line. This is the shape a CloudWatch Logs EMF ingestion +/// or a metric filter consumes; nothing here talks to AWS. +pub struct JsonLinesSink { + target: Target, + name: String, +} +enum Target { + Stderr, + Stdout, + File(std::path::PathBuf), +} +impl JsonLinesSink { + pub fn stderr() -> Self { + Self { + target: Target::Stderr, + name: "jsonlines:stderr".into(), + } + } + pub fn stdout() -> Self { + Self { + target: Target::Stdout, + name: "jsonlines:stdout".into(), + } + } + pub fn file(path: impl Into) -> Self { + let path = path.into(); + Self { + name: format!("jsonlines:{}", path.display()), + target: Target::File(path), + } + } +} +impl Sink for JsonLinesSink { + fn write(&mut self, record: &Value) -> Result<()> { + let mut line = serde_json::to_vec(record)?; + line.push(b'\n'); + match &self.target { + Target::Stderr => std::io::stderr().write_all(&line)?, + Target::Stdout => std::io::stdout().write_all(&line)?, + // Appended whole, so a concurrent reader never sees half a record. + Target::File(path) => OpenOptions::new() + .create(true) + .append(true) + .open(path)? + .write_all(&line)?, + } + Ok(()) + } + fn describe(&self) -> String { + self.name.clone() + } +} + +/// Builds records and hands them to a sink. +pub struct Metrics { + sink: Box, + environment: String, + math_env: String, + dropped: u64, +} +impl Metrics { + pub fn new(sink: Box, environment: &str, math_env: &str) -> Self { + Self { + sink, + environment: environment.to_owned(), + math_env: math_env.to_owned(), + dropped: 0, + } + } + /// `P026_METRICS` selects the sink: `off` (default), `stderr`, `stdout`, or + /// a filesystem path. `P026_ENVIRONMENT` is P-031's `Environment` dimension + /// and defaults to `synthetic`, never to `prod`. + /// + /// `off` is the default on purpose: writing records into a pipe that no + /// reader is draining blocks the writer once the pipe buffer fills, and a + /// coordinator must not stall because of its own telemetry. + pub fn from_env(config: &Config) -> Self { + let sink: Box = match config.metrics_sink.as_str() { + "off" => Box::new(NullSink), + "stdout" => Box::new(JsonLinesSink::stdout()), + "stderr" => Box::new(JsonLinesSink::stderr()), + path => Box::new(JsonLinesSink::file(path)), + }; + Self::new(sink, &config.environment, &config.math_env) + } + pub fn describe(&self) -> String { + self.sink.describe() + } + pub fn dropped(&self) -> u64 { + self.dropped + } + /// Emit one record. `context` is diagnostic only: it is a log property, not + /// a CloudWatch dimension, so it can carry a zid without creating the + /// per-conversation dimension P-031 forbids. + pub fn emit(&mut self, operation: &str, data: &[Datum], context: Value) { + if data.is_empty() { + return; + } + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_millis() as u64); + let mut record = json!({ + "schema": SCHEMA, + "_aws": { + "Timestamp": timestamp, + "CloudWatchMetrics": [{ + "Namespace": NAMESPACE, + "Dimensions": [["Environment", "MathEnv"]], + "Metrics": data.iter().map(|d| json!({"Name": d.name, "Unit": d.unit.as_str()})) + .collect::>(), + }], + }, + "Environment": self.environment, + "MathEnv": self.math_env, + "operation": operation, + "context": context, + }); + for d in data { + record[d.name] = json!(d.value); + } + if let Err(error) = self.sink.write(&record) { + self.dropped += 1; + tracing::warn!(error=%error, dropped = self.dropped, "metric record not written"); + } + } +} + +/// The declared catalog as JSON, for `polis-coordinator metrics` and for the +/// P-031 evidence file. Generated from [`CATALOG`], never hand-maintained. +pub fn catalog_json() -> Value { + json!({ + "schema": SCHEMA, + "namespace": NAMESPACE, + "dimensions": ["Environment", "MathEnv"], + "transport": "CloudWatch Embedded Metric Format records on a JSON-lines sink; no AWS client in this crate", + "p031_status": { + "coverage_claimed": [], + "not_implemented": ["A01 PollHealthy", "A02 PublishLagSeconds", "A03 ObserverHealthy"], + "note": "These are local diagnostics with no deployed publisher and no delivery proof. \ +A01 needs both poll loops to have succeeded, A02 needs initiated-but-unpublished work, and A03 needs \ +an independent observer; none of the series below is any of those. Scope: every gauge is scoped to \ +this shard and allowlist." + }, + "metrics": CATALOG.iter().map(|d| json!({ + "name": d.name, + "unit": d.unit.as_str(), + "statistic": d.statistic, + "p031_alarm": d.alarm, + "meaning": d.meaning, + })).collect::>(), + }) +} diff --git a/coordinator-rs/src/ordering.rs b/coordinator-rs/src/ordering.rs new file mode 100644 index 0000000000..7b28f15cda --- /dev/null +++ b/coordinator-rs/src/ordering.rs @@ -0,0 +1,122 @@ +//! The declared live source normalization, `polis-order/1`. +//! +//! Rev5 item 1: a live full-rebuild profile may declare the content +//! normalization `(tid,pid,created_ms,semantic_vote,weight)`, where +//! `semantic_vote = raw_vote * storage_agree_value`. The polarity constant is +//! therefore a **declared contract term**, not a literal spelled into one SQL +//! string: the coordinator builds its ORDER BY from these terms and ships the +//! same manifest to the worker, so both sides read one declaration. Ordering on +//! the raw sign instead would break the polarity property (P-023, D3). +use crate::store::digest; +use anyhow::Result; +use serde_json::{Value, json}; + +pub const SCHEMA: &str = "polis-order/1"; +pub const PROFILE: &str = "live-tid-pid-created-semantic-value-weight/1"; +/// Declared configuration input that carries the agree convention. +pub const PARAMETER: &str = "storage_agree_value"; +/// The bind position `source()` gives `PARAMETER`; asserted by the query builder. +pub const PARAMETER_BINDING: &str = "$2"; +pub const SEMANTIC_VOTE: &str = "raw_vote * storage_agree_value"; + +pub struct Term { + pub name: &'static str, + /// SQL over `votes`; only `PARAMETER_BINDING` may be a bound parameter. + pub expression: &'static str, + pub nulls: &'static str, + pub source: &'static str, +} + +pub const TERMS: &[Term] = &[ + Term { + name: "tid", + expression: "tid", + nulls: "", + source: "votes.tid", + }, + Term { + name: "pid", + expression: "pid", + nulls: "", + source: "votes.pid", + }, + Term { + name: "created_ms", + expression: "created", + nulls: "", + source: "votes.created", + }, + Term { + name: "semantic_vote", + expression: "(vote::bigint*$2::bigint)", + nulls: "", + source: SEMANTIC_VOTE, + }, + Term { + name: "weight_x_32767", + expression: "weight_x_32767", + nulls: " NULLS FIRST", + source: "votes.weight_x_32767", + }, +]; + +/// The declared ORDER BY, assembled from the terms above. +pub fn order_by() -> String { + TERMS + .iter() + .map(|t| format!("{}{}", t.expression, t.nulls)) + .collect::>() + .join(",") +} + +/// The terms whose only purpose is to break an otherwise ambiguous tie, i.e. +/// everything after the `(tid,pid,created_ms)` census key. +pub fn tiebreak_terms() -> Vec<&'static str> { + TERMS.iter().skip(3).map(|t| t.name).collect() +} + +fn algorithm(storage_agree_value: i64) -> Value { + json!({"schema":SCHEMA,"profile":PROFILE,"parameter":PARAMETER, + "storage_agree_value":storage_agree_value,"semantic_vote":SEMANTIC_VOTE, + "terms":TERMS.iter().map(|t|json!({"name":t.name,"source":t.source, + "nulls":if t.nulls.is_empty(){"default"}else{"first"}})).collect::>(), + "order_by":order_by()}) +} + +/// Pins the ordering algorithm; independent of any one conversation's data. +pub fn algorithm_digest(storage_agree_value: i64) -> Result { + Ok(digest(&serde_json::to_vec(&algorithm(storage_agree_value))?)) +} + +/// The declared normalization plus this conversation's `equal_time_census`. +/// This exact object is both the coordinator's ordering declaration and the +/// worker manifest's `ordering` value. +pub fn manifest(storage_agree_value: i64, census: Value) -> Result { + let mut value = algorithm(storage_agree_value); + value["algorithm_digest"] = json!(algorithm_digest(storage_agree_value)?); + value["equal_time_census"] = census; + Ok(value) +} + +/// Rev5: record ambiguous tied maxima rather than silently resolving them. +/// A group is a set of rows sharing `(tid,pid,created_ms)` whose relative order +/// is decided only by the trailing tiebreak terms. `rows` must already be in +/// the declared order. +pub fn census(rows: &[Value]) -> Value { + let key = |v: &Value| (v["tid"].clone(), v["pid"].clone(), v["created"].clone()); + let (mut groups, mut tied) = (0usize, 0usize); + let mut index = 0usize; + while index < rows.len() { + let mut end = index + 1; + while end < rows.len() && key(&rows[end]) == key(&rows[index]) { + end += 1; + } + if end - index > 1 { + groups += 1; + tied += end - index; + } + index = end; + } + json!({"schema":"polis-census/1","key":["tid","pid","created_ms"], + "resolved_by":tiebreak_terms(),"tied_groups":groups,"tied_rows":tied}) +} diff --git a/coordinator-rs/src/probe.rs b/coordinator-rs/src/probe.rs new file mode 100644 index 0000000000..f7c1c44937 --- /dev/null +++ b/coordinator-rs/src/probe.rs @@ -0,0 +1,185 @@ +//! CO01 incremental discovery: a cheap per-conversation change probe in front +//! of the authoritative full snapshot, and the durable per-zid cursor that +//! bounds how long the fast path may be trusted. +//! +//! The contract is explicit about what this may and may not be. Rev5: +//! "Count/max(created)/max(modified) are optimization hints, not completeness +//! proof: unchanged aggregate maxima can hide changed state or a late commit. +//! Keep the authoritative reconciliation path unless a separately proven change +//! token covers every relevant transaction; never make a weak hint the only +//! rebuild gate." So this probe is a *filter*, never a gate: +//! +//! 1. A probe that **differs** always forces the authoritative full snapshot. +//! 2. A probe that **matches** skips the full snapshot only while this +//! conversation's last authoritative reconciliation is younger than +//! `P026_RECONCILE_SECONDS`. Every conversation is therefore fully +//! reconciled at a bounded age no matter what the hint says, and CO05's +//! complete fair sweep stays the unconditional backstop. +//! 3. `P026_INCREMENTAL=0` disables the fast path entirely and restores the +//! original behaviour: every pass takes the full authoritative snapshot. +//! 4. The probe is captured **before** the snapshot it certifies, and the +//! database timestamp it carries is the one recorded as `reconciled_at`. +//! A row that commits between the probe and the snapshot changes the *next* +//! probe, so a concurrent write can never be swallowed by a stored later +//! probe; and because the stored age runs from *before* the source read +//! rather than from the end of publication, a long compute cannot reset the +//! advertised source age (Rev7 CO01: "a completion timestamp must not +//! masquerade as the age of the source observed"). +//! 5. The probe carries `ordering::algorithm_digest`, so the declared source +//! normalization and its storage agree-convention constant are part of the +//! change token: flipping the convention invalidates every conversation +//! even though no source row moved. +//! +//! `OldestReconciliationAgeSeconds` is the metric that makes the whole scheme +//! auditable, and it is why the metrics in this prototype are not decoration: +//! the incremental fast path is only sound while that age stays bounded. +use crate::{ordering, store::PgStore}; +use anyhow::{Result, ensure}; +use serde_json::Value; +use std::time::{Duration, SystemTime}; + +/// Pinned so a stored probe from an older shape is never compared as equal. +pub const SCHEMA: &str = "polis-source-probe/1"; + +/// One statement, therefore one snapshot, over the three source tables. +/// Aggregates only: no row is materialised and no payload is detoasted. +const PROBE_SQL: &str = "SELECT clock_timestamp(), jsonb_build_object( + 'schema',$3::text, + 'votes',(SELECT jsonb_build_object('n',count(*),'max_created',max(created), + 'min_created',min(created),'sum_created',COALESCE(sum(created),0), + 'sum_vote',COALESCE(sum(vote),0),'sum_weight',COALESCE(sum(weight_x_32767),0)) + FROM votes WHERE zid=$1), + 'comments',(SELECT jsonb_build_object('n',count(*),'max_modified',max(modified), + 'sum_modified',COALESCE(sum(modified),0),'sum_mod',COALESCE(sum(mod),0), + 'meta',count(*) FILTER (WHERE is_meta)) + FROM comments WHERE zid=$1), + 'participants',(SELECT jsonb_build_object('n',count(*),'sum_mod',COALESCE(sum(mod),0)) + FROM participants WHERE zid=$1), + 'ordering_algorithm',$2::text)"; + +/// Aggregate CO01/CO06 gauges for one namespace and shard. +#[derive(Debug, Clone, Copy)] +pub struct Backlog { + /// Conversations never reconciled, or overdue for reconciliation. + pub overdue: i64, + /// CO01 scan age. For a never-reconciled conversation the age runs from + /// `conversations.created`, so a conversation the coordinator has never + /// looked at cannot hide behind an empty table. + pub oldest_reconciliation: Duration, + /// Conversations currently in durable backoff. Scoped to this shard and + /// allowlist, exactly like the source-age gauges: a failure this process + /// would never attempt is not this process's backlog (Rev7 observability). + pub failures: i64, + /// CO06 oldest unrepaired age. + pub oldest_unrepaired: Duration, +} + +/// A probe, with the database time at which it — and therefore, conservatively, +/// the source snapshot it precedes — was observed. +pub struct Probe { + pub value: Value, + pub observed_at: SystemTime, +} + +impl PgStore { + /// The cheap change hint for one conversation, stamped with database time + /// taken in the same statement and therefore before the source read. + pub fn probe(&mut self, zid: i32) -> Result { + let algorithm = ordering::algorithm_digest(self.config.storage_agree_value)?; + let row = self + .client + .query_one(PROBE_SQL, &[&zid, &algorithm, &SCHEMA])?; + Ok(Probe { + observed_at: row.get(0), + value: row.get(1), + }) + } + + /// The stored probe and the age of the authoritative snapshot it certifies. + pub fn reconciliation(&mut self, zid: i32) -> Result> { + let row = self.client.query_opt( + "SELECT source_probe,EXTRACT(EPOCH FROM clock_timestamp()-reconciled_at)::float8 + FROM coordinator_reconciliation WHERE math_env=$1 AND zid=$2", + &[&self.config.math_env, &zid], + )?; + Ok(row.map(|r| { + ( + r.get::<_, Value>(0), + Duration::from_secs_f64(r.get::<_, f64>(1).max(0.0)), + ) + })) + } + + /// Record that the authoritative snapshot was taken, with the probe that + /// was observed **before** it and the database time of that observation — + /// not the time this call happens, which is after compute and publication. + /// Its own small transaction: this is a discovery hint, and CO04 forbids + /// adding it to the publication transaction's lock set. Parent-first + /// ordering still applies. + pub fn record_reconciliation(&mut self, zid: i32, probe: &Probe) -> Result<()> { + ensure!(probe.value["schema"] == SCHEMA, "unrecognised probe shape"); + let mut tx = self.client.transaction()?; + tx.query_one( + "SELECT zid FROM conversations WHERE zid=$1 FOR KEY SHARE", + &[&zid], + )?; + tx.execute( + "INSERT INTO coordinator_reconciliation(math_env,zid,reconciled_at,source_probe) + VALUES($1,$2,$4,$3) + ON CONFLICT(math_env,zid) DO UPDATE + SET reconciled_at=excluded.reconciled_at,source_probe=excluded.source_probe", + &[ + &self.config.math_env, + &zid, + &probe.value, + &probe.observed_at, + ], + )?; + tx.commit()?; + Ok(()) + } + + /// The CO01 backlog/scan-age and CO06 unrepaired-age gauges, scoped to this + /// namespace and this shard's candidate conversations. One bounded + /// aggregate statement; it reads no payload column. + pub fn backlog(&mut self) -> Result { + let c = &self.config; + let row = self.client.query_one( + "WITH mine AS ( + SELECT zid,created FROM conversations + WHERE zid % $2 = $3 + AND (cardinality($4::int[])=0 OR zid = ANY($4)) + ), state AS ( + SELECT m.zid, + COALESCE(EXTRACT(EPOCH FROM r.reconciled_at),m.created/1000.0) AS at, + r.reconciled_at IS NULL + OR r.reconciled_at <= clock_timestamp()-make_interval(secs=>$5::int) + AS overdue + FROM mine m + LEFT JOIN coordinator_reconciliation r + ON r.math_env=$1 AND r.zid=m.zid + ) + SELECT COALESCE(count(*) FILTER (WHERE overdue),0)::bigint, + COALESCE(GREATEST(EXTRACT(EPOCH FROM clock_timestamp())-min(at),0),0)::float8, + (SELECT count(*) FROM coordinator_failures f + WHERE f.math_env=$1 AND EXISTS(SELECT 1 FROM mine m WHERE m.zid=f.zid))::bigint, + (SELECT COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(f.first_failed_at)),0) + FROM coordinator_failures f + WHERE f.math_env=$1 AND EXISTS(SELECT 1 FROM mine m WHERE m.zid=f.zid))::float8 + FROM state", + &[ + &c.math_env, + &c.shard_count, + &c.shard_index, + &c.allowlist, + &c.reconcile_seconds, + ], + )?; + Ok(Backlog { + overdue: row.get(0), + oldest_reconciliation: Duration::from_secs_f64(row.get::<_, f64>(1).max(0.0)), + failures: row.get(2), + oldest_unrepaired: Duration::from_secs_f64(row.get::<_, f64>(3).max(0.0)), + }) + } +} diff --git a/coordinator-rs/src/reader.rs b/coordinator-rs/src/reader.rs new file mode 100644 index 0000000000..54bd6a85ef --- /dev/null +++ b/coordinator-rs/src/reader.rs @@ -0,0 +1,77 @@ +//! CO05 metadata-first reader: independent keyset sweep + paginated window. +//! Durable positions are hints. No durable payload cache is claimed: every page +//! is a cache miss in this command-line consumer, so it loads complete Bundles. +use crate::store::{Bundle, Current, PgStore, ResultsStore}; +use anyhow::{Result, ensure}; +use serde_json::{Value, json}; +impl PgStore { + pub fn reader_page(&mut self, consumer: &str) -> Result> { + ensure!( + !consumer.is_empty() && consumer.len() < 128, + "invalid consumer" + ); + let key = format!("reader-{consumer}"); + let position = self + .client + .query_opt( + "SELECT position FROM coordinator_cursors WHERE math_env=$1 AND consumer=$2", + &[&self.config.math_env, &key], + )? + .map(|r| r.get::<_, Value>(0)) + .unwrap_or(json!({})); + let high = position["high"].as_i64().unwrap_or(0); + let after_tick = position["after_tick"].as_i64().unwrap_or(-1); + let after_zid = position["after_zid"].as_i64().unwrap_or(0) as i32; + let sweep = position["sweep"].as_i64().unwrap_or(0) as i32; + let maximum = position["maximum"].as_i64().unwrap_or(high); + self.fault.hit("before_cursor", &position)?; + let fast = self.poll_changes( + high, + self.config.window, + (after_tick, after_zid), + self.config.page_size, + )?; + let maximum = fast + .iter() + .map(|m| m.caching_tick) + .max() + .unwrap_or(maximum) + .max(maximum); + let next_fast = if fast.len() == self.config.page_size as usize { + fast.last().map(|m| (m.caching_tick, m.zid)) + } else { + None + }; + self.fault.hit("before_sweep", &position)?; + let scanned = self.scan_current(sweep, self.config.page_size)?; + let next_sweep = if scanned.len() == self.config.page_size as usize { + scanned.last().map_or(0, |m| m.zid) + } else { + 0 + }; + self.fault.hit( + "after_sweep", + &json!({"rows":scanned,"next_sweep":next_sweep}), + )?; + let mut ids = std::collections::BTreeSet::new(); + let mut bundles = Vec::new(); + for meta in fast.iter().chain(scanned.iter()) { + if ids.insert(meta.zid) { + match self.load_current(meta.zid)? { + Current::Coherent(bundle) => { + ensure!(bundle.math_tick >= meta.math_tick, "generation regressed"); + bundles.push(*bundle); + } + _ => { + anyhow::bail!("reader found incomplete generation; retry page after repair") + } + } + } + } + let next = json!({"high":if next_fast.is_some(){high}else{maximum},"maximum":maximum, + "after_tick":next_fast.map_or(-1,|p|p.0),"after_zid":next_fast.map_or(0,|p|p.1),"sweep":next_sweep}); + self.client.execute("INSERT INTO coordinator_cursors(math_env,consumer,position) VALUES($1,$2,$3) ON CONFLICT(math_env,consumer) DO UPDATE SET position=excluded.position",&[&self.config.math_env,&key,&next])?; + self.fault.hit("after_cursor", &next)?; + Ok(bundles) + } +} diff --git a/coordinator-rs/src/store.rs b/coordinator-rs/src/store.rs new file mode 100644 index 0000000000..ae6047af68 --- /dev/null +++ b/coordinator-rs/src/store.rs @@ -0,0 +1,647 @@ +//! CO04 v0: parent -> lease -> ticks -> bidtopid -> ptptstats -> main. +use crate::{ + cache::WarmCache, + config::Config, + fault::Fault, + lease::{self, LeaseState}, + metrics::{Metrics, Tally}, +}; +use anyhow::{Result, ensure}; +use postgres::{Client, NoTls}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +pub fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +/// Integrity digest over JSONB numeric equality, NOT the harness semantic hash. +/// PostgreSQL discards negative zero and may render exponents as integers. +/// Expand decimal tokens and trim insignificant fractional zeroes before hashing. +pub fn storage_digest(value: &Value) -> Result { + fn number(text: &str) -> Result { + let (mantissa, exponent) = match text.split_once(['e', 'E']) { + Some((m, e)) => (m, e.parse::()?), + None => (text, 0), + }; + let negative = mantissa.starts_with('-'); + let unsigned = mantissa.trim_start_matches('-'); + let fractional = unsigned.split_once('.').map_or(0, |(_, f)| f.len()) as i32; + let mut digits = unsigned.replace('.', ""); + let scale = fractional - exponent; + if scale <= 0 { + digits.extend(std::iter::repeat_n('0', (-scale) as usize)); + } else { + while digits.len() <= scale as usize { + digits.insert(0, '0'); + } + digits.insert(digits.len() - scale as usize, '.'); + while digits.ends_with('0') { + digits.pop(); + } + if digits.ends_with('.') { + digits.pop(); + } + } + while digits.len() > 1 && digits.starts_with('0') && !digits.starts_with("0.") { + digits.remove(0); + } + if negative && digits != "0" { + digits.insert(0, '-'); + } + Ok(digits) + } + fn encode(v: &Value, out: &mut Vec) -> Result<()> { + match v { + Value::Number(n) => out.extend_from_slice(number(&n.to_string())?.as_bytes()), + Value::Array(a) => { + out.push(b'['); + for (i, v) in a.iter().enumerate() { + if i > 0 { + out.push(b','); + } + encode(v, out)?; + } + out.push(b']'); + } + Value::Object(m) => { + out.push(b'{'); + for (i, (k, v)) in m.iter().enumerate() { + if i > 0 { + out.push(b','); + } + serde_json::to_writer(&mut *out, k)?; + out.push(b':'); + encode(v, out)?; + } + out.push(b'}'); + } + _ => serde_json::to_writer(&mut *out, v)?, + } + Ok(()) + } + let mut bytes = Vec::new(); + encode(value, &mut bytes)?; + Ok(digest(&bytes)) +} + +/// Exact worker output, retained before JSONB normalization or re-encoding. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OriginalPayloads { + pub main: Vec, + pub bidtopid: Vec, + pub ptptstats: Vec, +} +impl OriginalPayloads { + fn hashes(&self) -> Value { + json!({"main":digest(&self.main),"bidtopid":digest(&self.bidtopid),"ptptstats":digest(&self.ptptstats)}) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Payloads { + pub originals: OriginalPayloads, + pub main: Value, + pub bidtopid: Value, + pub ptptstats: Value, +} +impl Payloads { + pub fn from_originals(originals: OriginalPayloads) -> Result { + Ok(Self { + main: crate::wire::parse(&originals.main)?, + bidtopid: crate::wire::parse(&originals.bidtopid)?, + ptptstats: crate::wire::parse(&originals.ptptstats)?, + originals, + }) + } + pub fn validate_originals(&self) -> Result<()> { + let parsed = Self::from_originals(self.originals.clone())?; + ensure!( + parsed.hashes()? == self.hashes()?, + "ORIGINAL_JSONB_MISMATCH" + ); + Ok(()) + } + + pub fn validate(&self, zid: i32) -> Result<()> { + for data in [&self.main, &self.bidtopid, &self.ptptstats] { + ensure!(data["zid"] == zid, "foreign payload identity"); + ensure!(data["lastVoteTimestamp"].is_i64(), "missing vote timestamp"); + ensure!( + data["lastVoteTimestamp"] == self.main["lastVoteTimestamp"], + "inconsistent payload timestamp" + ); + } + let ids = self.main["base-clusters"]["id"] + .as_array() + .ok_or_else(|| anyhow::anyhow!("missing base ids"))?; + let members = self.main["base-clusters"]["members"] + .as_array() + .ok_or_else(|| anyhow::anyhow!("missing base members"))?; + let bids = self.bidtopid["bidToPid"] + .as_array() + .ok_or_else(|| anyhow::anyhow!("missing bid mapping"))?; + ensure!( + ids.len() == bids.len() && members == bids, + "positional mapping mismatch" + ); + ensure!( + ids.windows(2).all(|w| w[0].as_i64() < w[1].as_i64()), + "unordered/duplicate base ids" + ); + let stats = self.ptptstats["ptptstats"] + .as_object() + .ok_or_else(|| anyhow::anyhow!("invalid stats"))?; + let len = stats + .get("pid") + .and_then(Value::as_array) + .map_or(0, Vec::len); + for values in stats.values() { + ensure!( + values.as_array().is_some_and(|v| v.len() == len), + "unequal stats columns" + ); + } + Ok(()) + } + pub fn hashes(&self) -> Result { + Ok(json!({"main":storage_digest(&self.main)?, + "bidtopid":storage_digest(&self.bidtopid)?, + "ptptstats":storage_digest(&self.ptptstats)?})) + } +} +#[derive(Debug, Serialize)] +pub struct Bundle { + pub payloads: Payloads, + pub math_tick: i64, + pub caching_tick: i64, + pub checkpoint: Value, + pub publisher_epoch: i64, + pub operation_id: String, +} +#[derive(Debug)] +pub enum Current { + Absent, + Inconsistent, + Coherent(Box), +} +#[derive(Debug, PartialEq)] +pub enum CommitReadback { + Own(i64), + Lost, +} +/// In-place storage cannot prove an overwritten operation ever committed. Fail +/// closed unless the current coherent generation is this exact attempt. +pub fn classify_commit( + current: &Current, + checkpoint: &Value, + epoch: i64, + operation_id: &str, + tick: i64, +) -> CommitReadback { + if let Current::Coherent(bundle) = current + && bundle.math_tick == tick + && bundle.publisher_epoch == epoch + && bundle.operation_id == operation_id + && bundle.checkpoint == *checkpoint + { + return CommitReadback::Own(tick); + } + CommitReadback::Lost +} +#[derive(Debug)] +pub struct CommitLost; +impl std::fmt::Display for CommitLost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("UNCERTAIN_COMMIT_LOST") + } +} +impl std::error::Error for CommitLost {} + +#[derive(Debug, PartialEq)] +pub enum Publication { + Committed(i64), + Conflict, + /// Ownership was refused inside the publication transaction, with the + /// distinguishable lease state that refused it. + Refused(LeaseState), +} +#[derive(Debug, Serialize)] +pub struct Metadata { + pub zid: i32, + pub math_tick: i64, + pub caching_tick: i64, +} +pub trait ResultsStore { + fn load_current(&mut self, zid: i32) -> Result; + fn publish( + &mut self, + zid: i32, + expected_tick: Option, + epoch: i64, + checkpoint: Value, + payload: &Payloads, + ) -> Result; + fn poll_changes( + &mut self, + high_water: i64, + window: i64, + after: (i64, i32), + limit: i64, + ) -> Result>; + fn scan_current(&mut self, after_zid: i32, limit: i64) -> Result>; +} +pub struct PgStore { + pub client: Client, + pub config: Config, + pub fault: Fault, + pub cache: WarmCache, + /// CO01/CO06 observability. Emission never fails an operation. + pub metrics: Metrics, + /// Counters for the pass currently in flight. + pub tally: Tally, + /// When the bounded backlog aggregate last ran. + pub gauged: Option, +} +impl PgStore { + pub fn connect(config: Config) -> Result { + Fault::reject_release_control()?; + let mut client = Client::connect(&config.database_url, NoTls)?; + client.batch_execute("SET statement_timeout='30s'; SET lock_timeout='5s'; SET application_name='p026-coordinator'")?; + let fault = Fault::new(&mut client, &config.math_env)?; + let cache = WarmCache::new(config.cache_capacity); + let metrics = Metrics::from_env(&config); + tracing::info!( + sink = metrics.describe(), + namespace = crate::metrics::NAMESPACE, + "metrics sink" + ); + Ok(Self { + client, + config, + fault, + cache, + metrics, + tally: Tally::default(), + gauged: None, + }) + } + /// Restore the primary connection after a terminated backend, so ownership + /// can still be released instead of being held for the rest of the window. + pub fn reconnect_if_closed(&mut self) -> Result<()> { + if self.client.is_closed() { + let mut client = Client::connect(&self.config.database_url, NoTls)?; + client.batch_execute("SET statement_timeout='30s'; SET lock_timeout='5s'; SET application_name='p026-coordinator'")?; + self.client = client; + } + Ok(()) + } + /// Explicit local migration command; never implicitly migrate on startup. + pub fn migrate(&mut self) -> Result<()> { + self.client + .batch_execute(include_str!("../migration.sql"))?; + Ok(()) + } + pub fn acquire(&mut self, zid: i32) -> Result> { + let c = &self.config; + let mut tx = self.client.transaction()?; + tx.query_one( + "SELECT zid FROM conversations WHERE zid=$1 FOR KEY SHARE", + &[&zid], + )?; + let rows = tx.query("INSERT INTO coordinator_leases (math_env,zid,owner_id,owner_epoch,expires_at) VALUES($1,$2,$3,1,clock_timestamp()+make_interval(secs=>$4::int)) ON CONFLICT(math_env,zid) DO UPDATE SET owner_id=excluded.owner_id,owner_epoch=coordinator_leases.owner_epoch+1,expires_at=excluded.expires_at WHERE coordinator_leases.expires_at<=clock_timestamp() OR coordinator_leases.owner_id=$3 RETURNING owner_epoch", &[&c.math_env,&zid,&c.owner,&c.lease_seconds])?; + let epoch = rows.first().map(|r| r.get(0)); + tx.commit()?; + Ok(epoch) + } + pub fn release(&mut self, zid: i32, epoch: i64) -> Result<()> { + let c = &self.config; + let mut tx = self.client.transaction()?; + tx.query_one( + "SELECT zid FROM conversations WHERE zid=$1 FOR KEY SHARE", + &[&zid], + )?; + tx.execute("UPDATE coordinator_leases SET expires_at=clock_timestamp() WHERE math_env=$1 AND zid=$2 AND owner_id=$3 AND owner_epoch=$4", &[&c.math_env,&zid,&c.owner,&epoch])?; + tx.commit()?; + Ok(()) + } + /// Authoritative classification of our ownership right now. + /// `None` means the lease is still ours and unexpired. + pub fn lease_state(&mut self, zid: i32, epoch: i64) -> Result> { + let row = self.client.query_opt("SELECT owner_id,owner_epoch,expires_at>clock_timestamp() FROM coordinator_leases WHERE math_env=$1 AND zid=$2", &[&self.config.math_env,&zid])?; + Ok(lease::classify(row.as_ref(), &self.config, epoch)) + } +} + +/// Metadata-only view of a persisted generation: no `data` column is read, so +/// nothing is detoasted. `complete` means all three companions exist at the same +/// tick as `math_ticks` and the provenance columns are present. +struct GenerationMeta { + tick: i64, + caching_tick: Option, + checkpoint: Option, + complete: bool, + epoch: Option, + operation_id: Option, +} + +impl PgStore { + fn generation_meta(&mut self, zid: i32) -> Result> { + let row = self.client.query_opt( + "SELECT t.math_tick,t.publisher_epoch,t.input_checkpoint, + m.math_tick,m.caching_tick,b.math_tick,p.math_tick,t.operation_id, + m.original_bytes IS NOT NULL AND m.original_sha256 IS NOT NULL + AND b.original_bytes IS NOT NULL AND b.original_sha256 IS NOT NULL + AND p.original_bytes IS NOT NULL AND p.original_sha256 IS NOT NULL + FROM math_ticks t + LEFT JOIN math_main m ON m.zid=t.zid AND m.math_env=t.math_env + LEFT JOIN math_bidtopid b ON b.zid=t.zid AND b.math_env=t.math_env + LEFT JOIN math_ptptstats p ON p.zid=t.zid AND p.math_env=t.math_env + WHERE t.math_env=$1 AND t.zid=$2", + &[&self.config.math_env, &zid], + )?; + let Some(r) = row else { + return Ok(None); + }; + let tick: i64 = r.get(0); + let checkpoint: Option = r.get(2); + let companions: [Option; 3] = [r.get(3), r.get(5), r.get(6)]; + let complete = r.get::<_, Option>(1).is_some() + && r.get::<_, Option>(7).is_some() + && r.get::<_, Option>(8) == Some(true) + && checkpoint.is_some() + && companions.iter().all(|t| *t == Some(tick)); + Ok(Some(GenerationMeta { + tick, + caching_tick: r.get(4), + checkpoint, + complete, + epoch: r.get(1), + operation_id: r.get(7), + })) + } + + /// CO06 quiet repair, cheaply. True only when a complete generation is + /// persisted for this conversation. The incremental fast path must never + /// skip past an incomplete or missing generation, however fresh its source + /// reconciliation record is: an absent companion or missing checkpoint + /// provenance has to be repaired without waiting for future input. + /// + /// This is metadata only. It cannot see a mutated payload; that is what the + /// reconciliation ceiling's authoritative `load_current` is for. + pub fn generation_is_complete(&mut self, zid: i32) -> Result { + Ok(self.generation_meta(zid)?.is_some_and(|g| g.complete)) + } + + /// Rev6 independent resident-cache integrity reconciliation. + /// + /// A bundle resident in the warm cache is **never** evidence that the + /// durable generation is still complete: a companion row can be deleted, + /// or its checkpoint changed, without the conversation's `math_tick` + /// moving, and the source fingerprint would then agree forever (F1). Every + /// pass that hits the cache re-verifies companion presence, every + /// companion's generation, and the committed checkpoint identity against + /// the store, again without selecting any payload column. + /// + /// Rev7 is explicit that this is *not* complete integrity reconciliation: + /// it cannot validate persisted payload content. The authoritative path + /// re-reads and re-hashes the payloads once per reconciliation ceiling. + /// + /// Returns false when the store contradicts the resident bundle in any way, + /// which evicts it and forces the ordinary repair path. + pub fn resident_is_intact(&mut self, zid: i32, bundle: &Bundle) -> Result { + Ok(self.generation_meta(zid)?.is_some_and(|g| { + g.complete + && g.epoch == Some(bundle.publisher_epoch) + && g.operation_id.as_ref() == Some(&bundle.operation_id) + && g.tick == bundle.math_tick + && g.caching_tick == Some(bundle.caching_tick) + && g.checkpoint.as_ref() == Some(&bundle.checkpoint) + })) + } + pub fn current_tick(&mut self, zid: i32) -> Result> { + Ok(self + .client + .query_opt( + "SELECT math_tick FROM math_ticks WHERE math_env=$1 AND zid=$2", + &[&self.config.math_env, &zid], + )? + .map(|r| r.get(0))) + } +} +impl ResultsStore for PgStore { + fn load_current(&mut self, zid: i32) -> Result { + let row = self.client.query_opt("SELECT m.data,b.data,p.data,m.math_tick,m.caching_tick,t.input_checkpoint,COALESCE(m.math_tick=b.math_tick AND m.math_tick=p.math_tick AND m.math_tick=t.math_tick AND t.publisher_epoch IS NOT NULL AND t.input_checkpoint IS NOT NULL,false),t.publisher_epoch,t.operation_id,m.original_bytes,b.original_bytes,p.original_bytes,m.original_sha256,b.original_sha256,p.original_sha256 FROM (SELECT zid FROM math_main WHERE math_env=$1 AND zid=$2 UNION SELECT zid FROM math_bidtopid WHERE math_env=$1 AND zid=$2 UNION SELECT zid FROM math_ptptstats WHERE math_env=$1 AND zid=$2 UNION SELECT zid FROM math_ticks WHERE math_env=$1 AND zid=$2) k LEFT JOIN math_main m ON m.zid=k.zid AND m.math_env=$1 LEFT JOIN math_bidtopid b ON b.zid=k.zid AND b.math_env=$1 LEFT JOIN math_ptptstats p ON p.zid=k.zid AND p.math_env=$1 LEFT JOIN math_ticks t ON t.zid=k.zid AND t.math_env=$1", &[&self.config.math_env,&zid])?; + let Some(r) = row else { + return Ok(Current::Absent); + }; + if !r.get::<_, bool>(6) { + return Ok(Current::Inconsistent); + } + let Some(operation_id) = r.get::<_, Option>(8) else { + return Ok(Current::Inconsistent); + }; + let raw: [Option>; 3] = [r.get(9), r.get(10), r.get(11)]; + let [Some(main), Some(bidtopid), Some(ptptstats)] = raw else { + return Ok(Current::Inconsistent); + }; + let originals = OriginalPayloads { + main, + bidtopid, + ptptstats, + }; + let hashes = originals.hashes(); + for (index, key) in [(12, "main"), (13, "bidtopid"), (14, "ptptstats")] { + if r.get::<_, Option>(index).as_deref() != hashes[key].as_str() { + return Ok(Current::Inconsistent); + } + } + let payloads = Payloads { + originals, + main: r.get(0), + bidtopid: r.get(1), + ptptstats: r.get(2), + }; + let checkpoint: Value = r.get(5); + if payloads.validate(zid).is_err() + || payloads.validate_originals().is_err() + || payloads.hashes()? != checkpoint["payload_digests"] + || hashes != checkpoint["original_digests"] + || checkpoint["operation_id"] != operation_id + || checkpoint["publisher_epoch"] != r.get::<_, i64>(7) + { + return Ok(Current::Inconsistent); + } + Ok(Current::Coherent(Box::new(Bundle { + payloads, + math_tick: r.get(3), + caching_tick: r.get(4), + checkpoint, + operation_id, + publisher_epoch: r.get(7), + }))) + } + fn publish( + &mut self, + zid: i32, + expected_tick: Option, + epoch: i64, + mut checkpoint: Value, + payload: &Payloads, + ) -> Result { + payload.validate(zid)?; + payload.validate_originals()?; + let operation_id = checkpoint["operation_id"] + .as_str() + .filter(|s| !s.is_empty() && s.len() <= 128) + .ok_or_else(|| anyhow::anyhow!("missing publication operation id"))? + .to_owned(); + checkpoint["publisher_epoch"] = json!(epoch); + checkpoint["original_digests"] = payload.originals.hashes(); + checkpoint["payload_digests"] = payload.hashes()?; // before ANY lock + let encoded = + [&payload.main, &payload.bidtopid, &payload.ptptstats].map(serde_json::to_string); + let [main, bid, stats] = encoded; + let (main, bid, stats) = (main?, bid?, stats?); + let stamp = payload.main["lastVoteTimestamp"] + .as_i64() + .ok_or_else(|| anyhow::anyhow!("invalid timestamp"))?; + let c = &self.config; + let mut tx = self.client.transaction()?; + tx.query_one( + "SELECT zid FROM conversations WHERE zid=$1 FOR KEY SHARE", + &[&zid], + )?; + let owned = tx.query_opt("SELECT owner_id,owner_epoch,expires_at>clock_timestamp() FROM coordinator_leases WHERE math_env=$1 AND zid=$2 FOR UPDATE", &[&c.math_env,&zid])?; + if let Some(state) = lease::classify(owned.as_ref(), c, epoch) { + return Ok(Publication::Refused(state)); + } + let context = json!({"zid":zid,"math_env":c.math_env,"epoch":epoch,"checkpoint":checkpoint, + "backend_pid":tx.query_one("SELECT pg_backend_pid()", &[])?.get::<_,i32>(0)}); + self.fault.hit("before_ticks", &context)?; + let current = tx + .query_opt( + "SELECT math_tick FROM math_ticks WHERE math_env=$1 AND zid=$2 FOR UPDATE", + &[&c.math_env, &zid], + )? + .map(|r| r.get::<_, i64>(0)); + if current != expected_tick { + return Ok(Publication::Conflict); + } + let tick = match current { + Some(n) => n + .checked_add(1) + .ok_or_else(|| anyhow::anyhow!("tick overflow"))?, + None => 0, + }; + tx.execute("INSERT INTO math_ticks(zid,math_env,math_tick,publisher_epoch,input_checkpoint,operation_id) VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT(zid,math_env) DO UPDATE SET math_tick=excluded.math_tick,publisher_epoch=excluded.publisher_epoch,input_checkpoint=excluded.input_checkpoint,operation_id=excluded.operation_id,modified=now_as_millis()", &[&zid,&c.math_env,&tick,&epoch,&checkpoint,&operation_id])?; + self.fault.hit("after_ticks", &context)?; + for (table, name, data, original) in [ + ( + "math_bidtopid", + "bidtopid", + bid, + &payload.originals.bidtopid, + ), + ( + "math_ptptstats", + "ptptstats", + stats, + &payload.originals.ptptstats, + ), + ] { + self.fault.hit(&format!("before_{name}"), &context)?; + tx.execute(&format!("INSERT INTO {table}(zid,math_env,math_tick,data,original_bytes,original_sha256) VALUES($1,$2,$3,$4::text::jsonb,$5,$6) ON CONFLICT(zid,math_env) DO UPDATE SET math_tick=excluded.math_tick,data=excluded.data,original_bytes=excluded.original_bytes,original_sha256=excluded.original_sha256,modified=now_as_millis()"), &[&zid,&c.math_env,&tick,&data,original,&checkpoint["original_digests"][name].as_str()])?; + self.fault.hit(&format!("after_{name}"), &context)?; + } + self.fault.hit("before_main", &context)?; + let cursor: i64 = tx + .query_one("SELECT nextval('coordinator_caching_tick')", &[])? + .get(0); + ensure!( + cursor <= 9_007_199_254_740_991, + "cursor exceeds exact Node integer range" + ); + tx.execute("INSERT INTO math_main(zid,math_env,math_tick,data,last_vote_timestamp,caching_tick,original_bytes,original_sha256) VALUES($1,$2,$3,$4::text::jsonb,$5,$6,$7,$8) ON CONFLICT(zid,math_env) DO UPDATE SET math_tick=excluded.math_tick,data=excluded.data,last_vote_timestamp=excluded.last_vote_timestamp,caching_tick=excluded.caching_tick,original_bytes=excluded.original_bytes,original_sha256=excluded.original_sha256,modified=now_as_millis()", &[&zid,&c.math_env,&tick,&main,&stamp,&cursor,&payload.originals.main,&checkpoint["original_digests"]["main"].as_str()])?; + self.fault.hit("after_main", &context)?; + self.fault.hit("before_commit", &context)?; + // Rev6 CO04 "remaining-lease final authorization under lock". The lease + // row is held `FOR UPDATE` for the whole publication, so the heartbeat + // cannot renew through it and a transferee cannot take an epoch while + // this transaction can still commit. The final authorization therefore + // re-reads owner, epoch and the **remaining** interval under that lock + // and requires a positive margin: reaching COMMIT with a lease about to + // elapse is refused rather than gambled on. This does not prove the + // COMMIT round trip finishes before expiry — nothing in-transaction can + // — so an uncertain COMMIT is still resolved by checkpoint identity + // below, never by a wall-clock deadline. + let final_row = tx.query_opt("SELECT owner_id,owner_epoch,expires_at>clock_timestamp(),EXTRACT(EPOCH FROM expires_at-clock_timestamp())::float8 FROM coordinator_leases WHERE math_env=$1 AND zid=$2 FOR UPDATE", &[&c.math_env,&zid])?; + if let Some(state) = lease::classify(final_row.as_ref(), c, epoch) { + return Ok(Publication::Refused(state)); + } + let remaining = final_row.as_ref().map_or(0.0, |r| r.get::<_, f64>(3)); + if remaining <= c.commit_margin_seconds { + tracing::error!( + zid, + remaining, + margin = c.commit_margin_seconds, + "remaining lease below the commit margin; rolling back" + ); + return Ok(Publication::Refused(LeaseState::Expired)); + } + let committed = tx.commit(); + if let Err(error) = committed { + // Lost COMMIT response: reconnect and compare immutable identity. + self.tally.publish_uncertain += 1; + self.client = Client::connect(&self.config.database_url, NoTls)?; + if let CommitReadback::Own(tick) = classify_commit( + &self.load_current(zid)?, + &checkpoint, + epoch, + &operation_id, + tick, + ) { + return Ok(Publication::Committed(tick)); + } + tracing::error!(%error, "uncertain COMMIT identity not observed"); + return Err(CommitLost.into()); + } + self.fault.hit("after_commit", &context)?; + Ok(Publication::Committed(tick)) + } + fn scan_current(&mut self, after_zid: i32, limit: i64) -> Result> { + ensure!((1..=1000).contains(&limit), "invalid page limit"); + let rows = self.client.query("SELECT zid,math_tick,caching_tick FROM math_main WHERE math_env=$1 AND zid>$2 ORDER BY zid LIMIT $3", &[&self.config.math_env,&after_zid,&limit])?; + Ok(rows + .iter() + .map(|r| Metadata { + zid: r.get(0), + math_tick: r.get(1), + caching_tick: r.get(2), + }) + .collect()) + } + fn poll_changes( + &mut self, + high_water: i64, + window: i64, + after: (i64, i32), + limit: i64, + ) -> Result> { + ensure!( + window > 0 && (1..=1000).contains(&limit), + "invalid polling parameters" + ); + let low = high_water.saturating_sub(window).max(0); + let rows = self.client.query("SELECT zid,math_tick,caching_tick FROM math_main WHERE math_env=$1 AND caching_tick >= $2 AND (caching_tick,zid)>($3,$4) ORDER BY caching_tick,zid LIMIT $5", &[&self.config.math_env,&low,&after.0,&after.1,&limit])?; + Ok(rows + .iter() + .map(|r| Metadata { + zid: r.get(0), + math_tick: r.get(1), + caching_tick: r.get(2), + }) + .collect()) + } +} diff --git a/coordinator-rs/src/wire.rs b/coordinator-rs/src/wire.rs new file mode 100644 index 0000000000..16bc29fba6 --- /dev/null +++ b/coordinator-rs/src/wire.rs @@ -0,0 +1,62 @@ +//! Strict JSON parser for untrusted worker frames and bulk files. +use serde::{ + Deserialize, Deserializer, + de::{self, MapAccess, SeqAccess, Visitor}, +}; +use serde_json::Value; +use std::fmt; +struct Strict(Value); +impl<'de> Deserialize<'de> for Strict { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> Visitor<'de> for V { + type Value = Strict; + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("JSON without duplicate keys") + } + fn visit_bool(self, v: bool) -> Result { + Ok(Strict(v.into())) + } + fn visit_i64(self, v: i64) -> Result { + Ok(Strict(v.into())) + } + fn visit_u64(self, v: u64) -> Result { + Ok(Strict(v.into())) + } + fn visit_f64(self, v: f64) -> Result { + serde_json::Number::from_f64(v) + .map(|n| Strict(Value::Number(n))) + .ok_or_else(|| E::custom("nonfinite JSON number")) + } + fn visit_str(self, v: &str) -> Result { + Ok(Strict(v.into())) + } + fn visit_string(self, v: String) -> Result { + Ok(Strict(v.into())) + } + fn visit_unit(self) -> Result { + Ok(Strict(Value::Null)) + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut out = Vec::new(); + while let Some(v) = seq.next_element::()? { + out.push(v.0); + } + Ok(Strict(Value::Array(out))) + } + fn visit_map>(self, mut map: A) -> Result { + let mut out = serde_json::Map::new(); + while let Some((k, v)) = map.next_entry::()? { + if out.insert(k, v.0).is_some() { + return Err(de::Error::custom("duplicate JSON key")); + } + } + Ok(Strict(Value::Object(out))) + } + } + deserializer.deserialize_any(V) + } +} +pub fn parse(bytes: &[u8]) -> Result { + serde_json::from_slice::(bytes).map(|v| v.0) +} diff --git a/coordinator-rs/tests/s1_admission.rs b/coordinator-rs/tests/s1_admission.rs new file mode 100644 index 0000000000..78caef1688 --- /dev/null +++ b/coordinator-rs/tests/s1_admission.rs @@ -0,0 +1,141 @@ +use polis_coordinator::{ + engine::{AdmissionError, CANDIDATE_SCHEMA, ENGINE_VERSION, admit_candidate}, + store::{Bundle, CommitReadback, Current, OriginalPayloads, Payloads, classify_commit}, +}; +use serde_json::{Value, json}; +fn requested() -> Value { + json!({"candidate_schema":CANDIDATE_SCHEMA,"engine_version":ENGINE_VERSION, + "input_digest":"a".repeat(64),"schedule_digest":"b".repeat(64),"operation_id":"synthetic-op"}) +} +#[test] +fn matching_candidate_is_admitted() { + assert_eq!(admit_candidate(&requested(), &requested()), Ok(())); +} +macro_rules! refusal { + ($name:ident, $key:literal, $value:expr, $reason:ident) => { + #[test] + fn $name() { + let mut actual = requested(); + actual[$key] = json!($value); + assert_eq!( + admit_candidate(&actual, &requested()), + Err(AdmissionError::$reason) + ); + } + }; +} +refusal!( + reserved_schema_refused, + "candidate_schema", + "polis-input/1", + Schema +); +refusal!( + engine_version_refused, + "engine_version", + "foreign/1", + Engine +); +refusal!(input_digest_refused, "input_digest", "0".repeat(64), Input); +refusal!( + schedule_digest_refused, + "schedule_digest", + "0".repeat(64), + Schedule +); +refusal!(operation_refused, "operation_id", "other-op", Operation); +refusal!(unknown_field_refused, "extra", "field", Malformed); +refusal!(wrong_type_refused, "engine_version", 1, Malformed); +#[test] +fn missing_and_malformed_identity_refused() { + for actual in [ + Value::Null, + json!([]), + json!({}), + json!({"candidate_schema":CANDIDATE_SCHEMA}), + ] { + assert_eq!( + admit_candidate(&actual, &requested()), + Err(AdmissionError::Malformed) + ); + } +} +fn current(epoch: i64, operation: &str) -> Current { + Current::Coherent(Box::new(Bundle { + payloads: Payloads::from_originals(OriginalPayloads { + main: b"{}".to_vec(), + bidtopid: b"{}".to_vec(), + ptptstats: b"{}".to_vec(), + }) + .unwrap_or_else(|e| panic!("{e}")), + publisher_epoch: epoch, + operation_id: operation.into(), + math_tick: 0, + caching_tick: 1, + checkpoint: json!({"synthetic":"identical deterministic checkpoint"}), + })) +} +#[test] +fn uncertain_commit_requires_own_epoch_even_at_identical_tick_and_checkpoint() { + let checkpoint = json!({"synthetic":"identical deterministic checkpoint"}); + assert_eq!( + classify_commit(¤t(7, "op"), &checkpoint, 7, "op", 0), + CommitReadback::Own(0) + ); + assert_eq!( + classify_commit(¤t(8, "op"), &checkpoint, 7, "op", 0), + CommitReadback::Lost + ); +} +#[test] +fn uncertain_commit_requires_own_operation_tick_and_checkpoint() { + let checkpoint = json!({"synthetic":"identical deterministic checkpoint"}); + assert_eq!( + classify_commit(¤t(7, "other"), &checkpoint, 7, "op", 0), + CommitReadback::Lost + ); + assert_eq!( + classify_commit(¤t(7, "op"), &checkpoint, 7, "op", 1), + CommitReadback::Lost + ); + assert_eq!( + classify_commit(¤t(7, "op"), &json!({}), 7, "op", 0), + CommitReadback::Lost + ); + assert_eq!( + classify_commit(&Current::Inconsistent, &checkpoint, 7, "op", 0), + CommitReadback::Lost + ); +} + +#[test] +fn checkpoint_envelope_rejects_unknown_missing_and_foreign_fields() -> anyhow::Result<()> { + use polis_coordinator::engine::admit_checkpoint; + let identity = + json!({"fixture_id":1,"run_id":"r","session_id":"s","checkpoint_id":"c","compute_id":"m"}); + let mut candidate = identity.clone(); + for (key, value) in json!({"schema":"polis-candidate-checkpoint/1","protocol":"polis-engine/1", + "profile":"candidate-profile","admission":requested(),"output_schema":"polis-candidate-math-output/1", + "state_schema":"rebuild-prefix/1","persistence":false,"math_input_cursors":{}, + "observed_state_cursors":{},"files":{"main":{},"bidtopid":{},"ptptstats":{},"restore":{}}}) + .as_object().ok_or_else(|| anyhow::anyhow!("test object"))? { candidate[key] = value.clone(); } + admit_checkpoint(&candidate, &requested(), &identity)?; + for (key, value, reason) in [ + ("unexpected", json!(1), AdmissionError::Malformed), + ("fixture_id", json!(2), AdmissionError::Operation), + ( + "schema", + json!("polis-checkpoint/1"), + AdmissionError::Malformed, + ), + ] { + let mut bad = candidate.clone(); + bad[key] = value; + let error = admit_checkpoint(&bad, &requested(), &identity) + .err() + .ok_or_else(|| anyhow::anyhow!("accepted corruption"))?; + assert_eq!(error.downcast_ref::(), Some(&reason)); + } + assert!(admit_checkpoint(&json!({}), &requested(), &identity).is_err()); + Ok(()) +} diff --git a/coordinator-rs/tests/validation.rs b/coordinator-rs/tests/validation.rs new file mode 100644 index 0000000000..ed79bc60cb --- /dev/null +++ b/coordinator-rs/tests/validation.rs @@ -0,0 +1,354 @@ +use anyhow::Result; +use polis_coordinator::store::{Payloads, digest}; +use serde_json::json; +fn payload() -> Payloads { + Payloads { + originals: polis_coordinator::store::OriginalPayloads { + main: vec![], + bidtopid: vec![], + ptptstats: vec![], + }, + main: json!({"zid":1,"lastVoteTimestamp":0,"base-clusters":{"id":[10,20],"members":[[1],[2]]}}), + bidtopid: json!({"zid":1,"lastVoteTimestamp":0,"bidToPid":[[1],[2]]}), + ptptstats: json!({"zid":1,"lastVoteTimestamp":0,"ptptstats":{"pid":[1,2],"gid":[0,1]}}), + } +} +#[test] +fn valid_mapping() -> Result<()> { + payload().validate(1) +} +#[test] +fn reject_swapped_buckets() { + let mut p = payload(); + p.bidtopid["bidToPid"] = json!([[2], [1]]); + assert!(p.validate(1).is_err()); +} +#[test] +fn reject_foreign_namespace_identity() { + assert!(payload().validate(2).is_err()); +} +#[test] +fn reject_duplicate_base_ids() { + let mut p = payload(); + p.main["base-clusters"]["id"] = json!([10, 10]); + assert!(p.validate(1).is_err()); +} +#[test] +fn reject_different_timestamps() { + let mut p = payload(); + p.ptptstats["lastVoteTimestamp"] = json!(1); + assert!(p.validate(1).is_err()); +} +#[test] +fn reject_unequal_stat_columns() { + let mut p = payload(); + p.ptptstats["ptptstats"]["gid"] = json!([0]); + assert!(p.validate(1).is_err()); +} +#[test] +fn hashes_cover_companions() -> Result<()> { + let p = payload(); + let mut other = payload(); + other.ptptstats["ptptstats"]["gid"] = json!([1, 0]); + assert_ne!(p.hashes()?, other.hashes()?); + Ok(()) +} +#[test] +fn ordinary_empty_prefix_hash() { + assert_eq!( + digest(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); +} +#[test] +fn jsonb_digest_ignores_numeric_spelling_only() -> Result<()> { + use polis_coordinator::store::storage_digest; + assert_eq!( + storage_digest(&json!([-0.0, 1.0, 1e19]))?, + storage_digest(&serde_json::from_str("[0,1,10000000000000000000]")?)? + ); + assert_ne!(storage_digest(&json!([1]))?, storage_digest(&json!(["1"]))?); + Ok(()) +} +#[test] +fn strict_wire_rejects_duplicate_keys_and_nonfinite() { + for bytes in [ + br#"{"ok":false,"ok":true}"#.as_slice(), + br#"{"nested":{"a":1,"a":2}}"#, + b"[NaN]", + ] { + assert!(polis_coordinator::wire::parse(bytes).is_err()); + } +} + +// The declared source normalization (`polis-order/1`), Rev5 item 1: the live +// tie key is over the SEMANTIC vote, and that is a contract term, not a literal. +#[test] +fn declared_tie_key_is_over_the_semantic_vote() -> Result<()> { + use polis_coordinator::ordering; + let names: Vec<_> = ordering::TERMS.iter().map(|t| t.name).collect(); + assert_eq!( + names, + [ + "tid", + "pid", + "created_ms", + "semantic_vote", + "weight_x_32767" + ] + ); + let sql = ordering::order_by(); + // The polarity constant is bound, and the raw sign is never ordered on. + assert!(sql.contains(&format!( + "vote::bigint*{}::bigint", + ordering::PARAMETER_BINDING + ))); + assert!(!sql.split(',').any(|term| term.trim() == "vote")); + assert!(sql.ends_with("weight_x_32767 NULLS FIRST")); + let census = json!({"tied_groups":0}); + let manifest = ordering::manifest(-1, census.clone())?; + assert_eq!(manifest["semantic_vote"], "raw_vote * storage_agree_value"); + assert_eq!(manifest["parameter"], ordering::PARAMETER); + assert_eq!(manifest["storage_agree_value"], -1); + assert_eq!(manifest["order_by"], sql); + assert_eq!(manifest["equal_time_census"], census); + // The declaration is polarity-bound: the agree convention is part of the + // pinned ordering algorithm, so the two conventions cannot share a digest. + assert_ne!( + ordering::algorithm_digest(-1)?, + ordering::algorithm_digest(1)? + ); + Ok(()) +} + +#[test] +fn census_records_only_ambiguous_tied_groups() { + use polis_coordinator::ordering::census; + let row = |tid: i64, pid: i64, created: i64| json!({"tid":tid,"pid":pid,"created":created}); + let none = census(&[row(1, 1, 10), row(1, 2, 10), row(2, 1, 10)]); + assert_eq!(none["tied_groups"], 0); + assert_eq!(none["tied_rows"], 0); + let tied = census(&[row(1, 1, 10), row(1, 1, 10), row(1, 2, 11), row(1, 2, 11)]); + assert_eq!(tied["tied_groups"], 2); + assert_eq!(tied["tied_rows"], 4); + assert_eq!(tied["key"], json!(["tid", "pid", "created_ms"])); + assert_eq!( + tied["resolved_by"], + json!(["semantic_vote", "weight_x_32767"]) + ); +} + +fn bundle(tick: i64) -> Box { + Box::new(polis_coordinator::store::Bundle { + payloads: payload(), + publisher_epoch: 1, + operation_id: "synthetic-operation".into(), + math_tick: tick, + caching_tick: tick, + checkpoint: json!({"source_fingerprint": "synthetic"}), + }) +} + +#[test] +fn warm_cache_never_returns_a_stale_generation() -> Result<()> { + use polis_coordinator::{cache::WarmCache, fault::Fault}; + let fault = Fault::default(); + let mut cache = WarmCache::new(2); + cache.insert(&fault, 7, bundle(4))?; + // Another writer advanced the generation: the entry is a miss, not a + // silently reused prior, and it is dropped rather than kept. + assert!(cache.take(7, Some(5)).is_none()); + assert!(!cache.contains(7)); + cache.insert(&fault, 7, bundle(4))?; + assert!(cache.take(7, None).is_none()); + cache.insert(&fault, 7, bundle(4))?; + assert!(cache.take(7, Some(4)).is_some()); + assert!(cache.is_empty()); + Ok(()) +} + +#[test] +fn warm_cache_evicts_least_recently_used_within_capacity() -> Result<()> { + use polis_coordinator::{cache::WarmCache, fault::Fault}; + let fault = Fault::default(); + let mut cache = WarmCache::new(2); + for zid in [1, 2] { + cache.insert(&fault, zid, bundle(0))?; + } + // Re-inserting 1 makes it most recent, so 2 is the victim. + cache.insert(&fault, 1, bundle(0))?; + cache.insert(&fault, 3, bundle(0))?; + assert_eq!(cache.len(), 2); + assert!(cache.contains(1) && cache.contains(3) && !cache.contains(2)); + let mut disabled = WarmCache::new(0); + disabled.insert(&fault, 1, bundle(0))?; + assert!(disabled.is_empty()); + Ok(()) +} + +// --- CO01 metrics: the record shape P-031 can consume --- + +#[derive(Clone, Default)] +struct Captured(std::sync::Arc>>); +impl polis_coordinator::metrics::Sink for Captured { + fn write(&mut self, record: &serde_json::Value) -> Result<()> { + match self.0.lock() { + Ok(mut records) => { + records.push(record.clone()); + Ok(()) + } + Err(_) => anyhow::bail!("poisoned"), + } + } + fn describe(&self) -> String { + "captured".into() + } +} +fn captured(sink: &Captured) -> Vec { + match sink.0.lock() { + Ok(records) => records.clone(), + Err(_) => panic!("poisoned capture"), + } +} + +#[test] +fn metric_record_carries_only_the_two_permitted_dimensions() { + use polis_coordinator::metrics::{Metrics, NAMESPACE, count, seconds}; + let sink = Captured::default(); + let mut metrics = Metrics::new(Box::new(sink.clone()), "synthetic", "rustproto"); + metrics.emit( + "source_pass", + &[ + count("SourcePassPublished", 2u32), + seconds("SourcePassSeconds", std::time::Duration::from_millis(1500)), + ], + json!({"zid": 7}), + ); + let records = captured(&sink); + assert_eq!(records.len(), 1); + let r = &records[0]; + assert_eq!(r["_aws"]["CloudWatchMetrics"][0]["Namespace"], NAMESPACE); + // P-031: "Math uses only fixed Environment=prod, MathEnv=prod ... No + // conversation/job/report/run/instance dimensions." + assert_eq!( + r["_aws"]["CloudWatchMetrics"][0]["Dimensions"], + json!([["Environment", "MathEnv"]]) + ); + assert_eq!(r["Environment"], "synthetic"); + assert_eq!(r["MathEnv"], "rustproto"); + assert_eq!(r["SourcePassPublished"], 2.0); + assert_eq!(r["SourcePassSeconds"], 1.5); + // The zid is a log property, never a dimension. + assert_eq!(r["context"]["zid"], 7); +} + +#[test] +fn every_emitted_metric_name_is_declared_in_the_catalog() { + use polis_coordinator::metrics::{CATALOG, Metrics, Tally}; + let sink = Captured::default(); + let mut metrics = Metrics::new(Box::new(sink.clone()), "synthetic", "rustproto"); + // The whole per-pass tally plus every per-zid and gauge name. + let mut data = Tally::default().data(std::time::Duration::from_secs(1), true); + data.extend([ + polis_coordinator::metrics::seconds( + "OldestReconciliationAgeSeconds", + std::time::Duration::from_secs(0), + ), + polis_coordinator::metrics::count("ReconciliationBacklogConversations", 0u32), + polis_coordinator::metrics::count("FailureBacklogConversations", 0u32), + polis_coordinator::metrics::seconds( + "OldestUnrepairedAgeSeconds", + std::time::Duration::from_secs(0), + ), + polis_coordinator::metrics::seconds( + "ConversationLatencySeconds", + std::time::Duration::from_secs(0), + ), + polis_coordinator::metrics::seconds("SourceReadSeconds", std::time::Duration::from_secs(0)), + polis_coordinator::metrics::seconds("ComputeSeconds", std::time::Duration::from_secs(0)), + polis_coordinator::metrics::seconds("PublishSeconds", std::time::Duration::from_secs(0)), + polis_coordinator::metrics::count("MetricsDropped", 0u32), + ]); + metrics.emit("all", &data, json!({})); + let records = captured(&sink); + let listed = match records[0]["_aws"]["CloudWatchMetrics"][0]["Metrics"].as_array() { + Some(list) => list.clone(), + None => panic!("no metric list in the record"), + }; + for entry in &listed { + assert!( + CATALOG + .iter() + .any(|d| Some(d.name) == entry["Name"].as_str()), + "undeclared metric {entry}" + ); + } + // and the catalog does not describe metrics nothing can emit + for declared in CATALOG { + assert!( + listed.iter().any(|e| e["Name"] == declared.name), + "catalog declares {} but no call site emits it", + declared.name + ); + } +} + +#[test] +fn a_failing_metric_sink_is_counted_and_never_propagates() { + use polis_coordinator::metrics::{Metrics, Sink, count}; + struct Broken; + impl Sink for Broken { + fn write(&mut self, _record: &serde_json::Value) -> Result<()> { + anyhow::bail!("sink is down") + } + fn describe(&self) -> String { + "broken".into() + } + } + let mut metrics = Metrics::new(Box::new(Broken), "synthetic", "rustproto"); + metrics.emit("t", &[count("SourcePassHealthy", 1u32)], json!({})); + assert_eq!(metrics.dropped(), 1, "a lost record must be visible"); +} + +#[test] +fn no_catalog_row_claims_a_p031_alarm_it_does_not_implement() { + // Rev7: none of these series is A01 PollHealthy, A02 PublishLagSeconds or + // A03 ObserverHealthy, so no row may be labelled with one. + for declared in polis_coordinator::metrics::CATALOG { + assert_eq!( + declared.alarm, "", + "{} claims P-031 {}", + declared.name, declared.alarm + ); + } + let catalog = polis_coordinator::metrics::catalog_json(); + assert_eq!(catalog["p031_status"]["coverage_claimed"], json!([])); + assert_eq!( + catalog["p031_status"]["not_implemented"], + json!([ + "A01 PollHealthy", + "A02 PublishLagSeconds", + "A03 ObserverHealthy" + ]) + ); + assert!( + !polis_coordinator::metrics::CATALOG + .iter() + .any(|d| d.name == "PollHealthy" + || d.name == "PublishLagSeconds" + || d.name == "ObserverHealthy"), + "a P-031 alarm name must not be reused for a different signal" + ); +} + +#[test] +fn catalog_metric_names_are_unique() { + let mut names: Vec<_> = polis_coordinator::metrics::CATALOG + .iter() + .map(|d| d.name) + .collect(); + names.sort_unstable(); + let before = names.len(); + names.dedup(); + assert_eq!(before, names.len(), "duplicate metric name in the catalog"); +} diff --git a/coordinator-rs/tools/node_reader.cjs b/coordinator-rs/tools/node_reader.cjs new file mode 100644 index 0000000000..fca1561398 --- /dev/null +++ b/coordinator-rs/tools/node_reader.cjs @@ -0,0 +1,120 @@ +/* + * CO08 / D4 harness: drive the REAL Node server reader against a published + * generation, in-process, with no HTTP layer and no mocks. + * + * It loads the actual server modules — `server/src/utils/pca.ts` (`getPca`, its + * LruCache keyed by `[math_env, zid]` and the gzip buffer `/api/v3/math/pca2` + * serves) and `server/src/utils/participants.ts` + * (`getBidIndexToPidMapping`, `getPidsForGid`) — and reports, per math_env: + * + * - `asJSON`: the exact string `participationInit` embeds as `response.pca` + * and the exact bytes `pca2` gzips; reported as sha256 plus its length. + * - `gzip`: sha256 of `asBufferOfGzippedJson`, the literal response body of + * `GET /api/v3/math/pca2` when no `keys` filter is given. + * - `keys`: the `_.pick(asPOJO, keys)` projection that route serves when + * `keys` IS given, so the other pca2 branch is covered too. + * - `mapping`: `getBidIndexToPidMapping`'s returned data. + * - `pids_for_gid`: the `getPca` + mapping join, per group. + * + * There is no `loadBundle` in the server today; PR #2703 scoped the existing + * reader by `[math_env, zid]` rather than introducing one. This harness + * therefore exercises the reader that actually exists. It is a real-consumer + * equality check, not a claim that CO04's Bundle rewrite has been done. + * + * Usage: node node_reader.cjs '' where the spec is + * {"zid":1,"envs":["python","rustproto"],"keys":["tids","n"],"gids":[0,1]} + * DATABASE_URL must be set. Writes one JSON object to stdout. + */ +"use strict"; + +const crypto = require("crypto"); +const path = require("path"); +const { createRequire } = require("module"); + +const server = path.resolve(__dirname, "../../server"); +// Resolve the server's own dependency tree, not this tool's directory. +const serverRequire = createRequire(path.join(server, "package.json")); + +serverRequire("ts-node").register({ + transpileOnly: true, + skipProject: true, + compilerOptions: { + module: "commonjs", + target: "es2020", + esModuleInterop: true, + allowJs: true, + resolveJsonModule: true, + skipLibCheck: true, + }, +}); + +const Config = serverRequire("./src/config").default; +const { getPca } = serverRequire("./src/utils/pca"); +const participants = serverRequire("./src/utils/participants"); +const _ = serverRequire("underscore"); + +const sha256 = (value) => crypto.createHash("sha256").update(value).digest("hex"); + +async function readNamespace(zid, mathEnv, keys, gids) { + // The sanctioned pattern from server/__tests__/integration/math-env-isolation.test.ts. + Config.mathEnv = mathEnv; + const item = await getPca(zid, undefined); + if (!item) { + return { math_env: mathEnv, present: false }; + } + const mapping = await participants.getBidIndexToPidMapping(zid, -1); + const joins = {}; + for (const gid of gids) { + try { + const pids = await participants.getPidsForGid(zid, gid, -1); + joins[gid] = pids; + } catch (error) { + joins[gid] = { error: String(error && error.message ? error.message : error) }; + } + } + const projection = JSON.stringify(_.pick(item.asPOJO, keys)); + return { + math_env: mathEnv, + present: true, + // The math_tick the route puts in the ETag. + etag_math_tick: item.asPOJO.math_tick, + asJSON_sha256: sha256(item.asJSON), + asJSON_bytes: Buffer.byteLength(item.asJSON, "utf-8"), + // Exactly what pca2 sends as the response body. + gzip_sha256: sha256(item.asBufferOfGzippedJson), + gzip_bytes: item.asBufferOfGzippedJson.length, + keys_projection_sha256: sha256(projection), + tids: item.asPOJO.tids, + n: item.asPOJO.n, + // The server's synthesized empty presentation stamps Date.now() here + // (pca.ts createEmptyPcaStructure), so it is reported rather than assumed. + last_vote_timestamp: item.asPOJO.lastVoteTimestamp, + repness_keys: Object.keys(item.repness || {}).sort(), + consensus_shape: { + agree: (item.consensus && item.consensus.agree && item.consensus.agree.length) || 0, + disagree: (item.consensus && item.consensus.disagree && item.consensus.disagree.length) || 0, + }, + mapping_is_error: mapping instanceof Error, + mapping_sha256: mapping instanceof Error ? null : sha256(JSON.stringify(mapping)), + mapping_math_tick: mapping instanceof Error ? null : mapping.math_tick, + bid_to_pid: mapping instanceof Error ? null : mapping.bidToPid, + pids_for_gid: joins, + }; +} + +async function main() { + const spec = JSON.parse(process.argv[2]); + const out = { zid: spec.zid, namespaces: [] }; + for (const mathEnv of spec.envs) { + out.namespaces.push(await readNamespace(spec.zid, mathEnv, spec.keys || [], spec.gids || [])); + } + process.stdout.write(JSON.stringify(out)); +} + +main().then( + () => process.exit(0), + (error) => { + process.stderr.write(String((error && error.stack) || error) + "\n"); + process.exit(1); + } +); diff --git a/coordinator-rs/tools/node_route_probe.cjs b/coordinator-rs/tools/node_route_probe.cjs new file mode 100644 index 0000000000..1e5025ce6e --- /dev/null +++ b/coordinator-rs/tools/node_route_probe.cjs @@ -0,0 +1,65 @@ +/* + * CO08/D4 first-generation route coverage, from Astra's round-2 review probe + * (cost-reduction/scripts/p2727-r2-node-review.cjs), committed unchanged in + * substance. + * + * It mounts the REAL `handle_GET_math_pca2` on a real Express app over loopback + * HTTP against the real reader, cache and PostgreSQL. Only the parameter + * middleware is replaced by a synthetic zid binding; the route, reader, cache, + * database and HTTP response implementation are the shipped ones. It does not + * cover application boot, auth, or the whole request-validation matrix. + * + * Its point is the caller argument: the route supplies math_tick = -1 for a + * request without a tick, while `getPca(zid, undefined)` takes a different + * branch. Usage: node node_route_probe.cjs ; DATABASE_URL required. + */ +'use strict'; +const {createRequire} = require('module'); +const path = require('path'); +const http = require('http'); +const zlib = require('zlib'); +const req = createRequire(path.join(process.argv[2], 'server/package.json')); +req('ts-node').register({transpileOnly:true, skipProject:true, compilerOptions:{ + module:'commonjs', target:'es2020', esModuleInterop:true, allowJs:true, + resolveJsonModule:true, skipLibCheck:true}}); +const pca = req('./src/utils/pca'); +const routes = req('./src/routes/math'); +const pg = req('./src/db/pg-query').default; +const express = req('express'); +async function main() { + const raw = await pg.queryP_readOnly('SELECT math_tick FROM math_main WHERE zid=1 AND math_env=$1', ['rustproto']); + const cold = await pca.getPca(1, undefined); + const app = express(); + app.set('env', 'production'); + app.get('/api/v3/math/pca2', (r,s) => { + // Only the parameter middleware is replaced by a synthetic zid binding. + r.p={zid:1, math_tick:undefined, ifNoneMatch:r.headers['if-none-match'], + keys:r.query.keys === undefined ? undefined : r.query.keys.split(',')}; + routes.handle_GET_math_pca2(r,s); + }); + const server = await new Promise(resolve => {const s=app.listen(0,'127.0.0.1',()=>resolve(s));}); + async function get(suffix='', headers={}) { + return new Promise((resolve,reject)=>{ + http.get({host:'127.0.0.1',port:server.address().port,path:'/api/v3/math/pca2'+suffix,headers},r=>{ + const chunks=[]; + r.on('data',b=>chunks.push(b));r.on('error',reject); + r.on('end',()=>{ + let b=Buffer.concat(chunks); + if(r.headers['content-encoding']==='gzip') b=zlib.gunzipSync(b); + const body=b.length?JSON.parse(b.toString()):null; + resolve({status:r.statusCode,etag:r.headers.etag,bodyTick:body&&body.math_tick, + contentEncoding:r.headers['content-encoding']||'identity',n:body&&body.n}); + }); + }).on('error',reject); + }); + } + try { + const whole=await get(); + const subset=await get('?keys=math_tick,n,tids'); + const conditional=await get('',{'If-None-Match':'"0"'}); + const warm=await pca.getPca(1,undefined); + console.log(JSON.stringify({pgTickType:typeof raw[0].math_tick,coldUndefinedPresent:!!cold, + whole,subset,conditional,warmUndefinedPresent:!!warm})); + } finally {await new Promise(resolve=>server.close(resolve));} +} +main().then(()=>process.exit(0),e=>{console.error(e);process.exit(1);}); diff --git a/coordinator-rs/tools/record_s1.py b/coordinator-rs/tools/record_s1.py new file mode 100644 index 0000000000..acab694b52 --- /dev/null +++ b/coordinator-rs/tools/record_s1.py @@ -0,0 +1,197 @@ +"""Record the local S1 gate as a PARTIAL O8 result; refuse missing or failed evidence. + +Run after the full commands in README/report, from any directory. The record +pins the **committed** repository inputs — source, migration, schemas, tests — +so `audit_stages.py` can re-verify it on any checkout of this commit. Build +outputs (`target/`) and run logs (`artifacts/`) are gitignored and therefore go +into an informational `run_pins` block that the audit does not gate on; those +hashes are true only of the producing host. + +Counts are derived from the run, never asserted against literals: adding a test +must not make this recorder fail. Deriving them is not the same as trusting them: +each cargo invocation's exit status is captured beside its log, and every suite in +a transcript must be a clean pass, so a failing suite after a passing one cannot +be silently omitted from the total and `ignored` tests are never counted as +passes. It records only this candidate campaign, never G01-G16 conformance or +Rust/Clojure equivalence. +""" +from collections import Counter +import hashlib +import importlib.metadata +import os +import platform +import json +from pathlib import Path +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + +ROOT = Path(__file__).resolve().parents[2] +ARTIFACTS = ROOT / "coordinator-rs/artifacts" +EVIDENCE = ROOT / "coordinator-rs/evidence" + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +CARGO_PROFILES = ("default", "fault") +CARGO_INVOCATIONS = ("cargo-default", "cargo-fault", "clippy-default", "clippy-fault", + "release", "fault-build") + + +def cargo_exit_status(artifacts, name): + """The captured exit status of one cargo invocation. + + The recorder must never certify a run it did not see succeed. Reading the + transcript is not enough: cargo prints a `test result:` line per suite, so a + passing suite followed by a failing one still leaves passing lines in the log. + Each gate command therefore writes its own `$?` next to its log, and a missing + status file is malformed evidence, not a pass. + """ + path = artifacts / f"s1-{name}.status" + assert path.exists(), f"no captured exit status for s1-{name}" + status = path.read_text().strip() + assert status == "0", f"s1-{name} exited {status}" + return status + + +def rust_test_counts(artifacts): + """Total passing Rust tests per feature profile, or refuse the transcript. + + Every suite in the transcript must be a clean pass. A failed suite after a + passing one fails the whole gate; `ignored` tests are not passes and are not + counted. Counts are derived from the run, never asserted against a literal. + """ + counts = {} + for profile in CARGO_PROFILES: + cargo_exit_status(artifacts, f"cargo-{profile}") + log = (artifacts / f"s1-cargo-{profile}.log").read_text() + results = re.findall(r"^test result: (\w+)\. (\d+) passed; (\d+) failed; (\d+) ignored", + log, re.M) + assert results, f"no cargo test result line in s1-cargo-{profile}.log" + assert len(results) == len(re.findall(r"^test result:", log, re.M)), \ + f"malformed cargo test result line in s1-cargo-{profile}.log" + for outcome, passed, failed, ignored in results: + assert outcome == "ok", f"failed cargo suite in s1-cargo-{profile}.log: {outcome}" + assert failed == "0", f"{failed} failed cargo tests in s1-cargo-{profile}.log" + assert ignored == "0", f"{ignored} ignored cargo tests in s1-cargo-{profile}.log" + assert not re.search(r"^error(:|\[)", log, re.M), f"cargo error in s1-cargo-{profile}.log" + assert "uncertain_commit_requires_own_epoch_even_at_identical_tick_and_checkpoint ... ok" in log + counts[profile] = sum(int(passed) for _, passed, _, _ in results) + assert counts["default"] == counts["fault"], counts + assert counts["default"], "no Rust tests ran" + return counts + + +def main(): + junit = ARTIFACTS / "s1-pytest.xml" + suites = ET.parse(junit).getroot().findall("testsuite") + assert suites and all(int(s.attrib[k]) == 0 for s in suites for k in ("failures", "errors", "skipped")) + cases = [case for suite in suites for case in suite.findall("testcase")] + counts = Counter(case.attrib["classname"].rsplit(".", 1)[-1] for case in cases) + # Derived, not hardcoded: require the S1 modules to have actually run and the + # suite to be non-empty. The exact totals are an observation of this run. + assert cases, "empty JUnit report" + for module in ("test_s1_identity", "test_adapter"): + assert counts[module], f"{module} contributed no cases" + rust_counts = rust_test_counts(ARTIFACTS) + for name in ("clippy-default", "clippy-fault", "release", "fault-build"): + cargo_exit_status(ARTIFACTS, name) + log = (ARTIFACTS / f"s1-{name}.log").read_text() + assert "Finished " in log and not re.search(r"^error(:|\[)", log, re.M), name + replay = json.loads((ARTIFACTS / "vw-equivalence.json").read_text()) + previous = json.loads((EVIDENCE / "vw-equivalence.json").read_text()) + assert len(replay["checkpoints"]) == 3 and replay["observer_errors"] == [] + assert all(c["deltas"] == [] and c["rust"] == c["python"] for c in replay["checkpoints"]) + assert [c["rust"] for c in replay["checkpoints"]] == [c["rust"] for c in previous["checkpoints"]] + for name in ("polarity-synthetic.json", "polarity-vw.json", "polarity-biodiversity.json", "polarity-rebuild-schedule.json", "semantic-tie-key.json"): + assert json.loads((ARTIFACTS / name).read_text()) == json.loads((EVIDENCE / name).read_text()), name + assert json.loads((EVIDENCE / "d4-node-reader.json").read_text())["differences"] == {} + (EVIDENCE / "vw-equivalence.json").write_bytes((ARTIFACTS / "vw-equivalence.json").read_bytes()) + # Committed inputs only: these hash the same on any checkout of this commit, + # which is what makes the recorded state re-verifiable by someone else. + paths = list((ROOT / "coordinator-rs/src").glob("*.rs")) + paths += list((ROOT / "coordinator-rs/tests").glob("*.rs")) + paths += list((ROOT / "coordinator-rs/schemas").glob("*.json")) + paths += list((ROOT / "delphi/tests/coordinator").glob("*.py")) + paths += list((ROOT / "delphi/polismath").rglob("*.py")) + paths += [ROOT / "coordinator-rs/evidence/python-requirements.txt", Path(__file__)] + paths += [ROOT / f"coordinator-rs/{name}" for name in ("migration.sql", "Cargo.toml", "Cargo.lock", "rust-toolchain.toml")] + ignored = ("coordinator-rs/target/", "coordinator-rs/artifacts/") + source_pins = {str(path.relative_to(ROOT)): sha(path) for path in sorted(set(paths))} + assert not [n for n in source_pins if n.startswith(ignored)], "gitignored path in source pins" + # Informational only. Binaries and logs live under gitignored directories, so + # a hash gate on them would hold on the producing host and nowhere else. + run_paths = [junit] + run_paths += [ROOT / f"coordinator-rs/{name}" for name in ("target/release/polis-coordinator", "target/fault/debug/polis-coordinator")] + run_paths += [ARTIFACTS / f"s1-{name}.log" for name in CARGO_INVOCATIONS] + [ARTIFACTS / "s1-pytest.log"] + run_paths += [ARTIFACTS / f"s1-{name}.status" for name in CARGO_INVOCATIONS] + compose_project = os.environ.get("COMPOSE_PROJECT_NAME") + postgres_port = os.environ.get("POLIS_RECOVERY_PG_PORT") or os.environ.get("RECOVERY_PG_PORT") + if postgres_port is None: + match = re.search(r":(\d+)/", os.environ.get("POLIS_TEST_POSTGRES_URL", "")) + postgres_port = match.group(1) if match else None + run_pins = dict(verifiable_on_producing_host_only=True, + junit=str(junit.relative_to(ROOT)), python_tests=len(cases), rust_tests=rust_counts, + compose_project=compose_project, postgres_port=int(postgres_port) if postgres_port else None, + platform=platform.platform(), + runtime=dict(python=platform.python_version(), packages={name: importlib.metadata.version(name) + for name in ("numpy", "scipy", "pandas", "scikit-learn", "psycopg2-binary", "pytest")}), + sha256={str(path.relative_to(ROOT)): sha(path) for path in sorted(set(run_paths)) if path.exists()}) + closure = dict(id="O8", state="PARTIAL", + scope="P-026 step-4 S1 candidate identity and original-byte custody", + g01_g16_certified=False, rust_equals_clojure_claimed=False, + candidate_schema="polis-candidate-input/1", engine_version="python-conversation/p026-s1", + recorded=[ + "distinct candidate schema id and strict adapter admission on both sides", + "original worker bytes and their digests stored atomically with the JSONB, with decoded original/JSONB correspondence checked on read", + "worker/engine identity re-verified at initialization and at snapshot", + "operation id and publisher epoch bound into the uncertain-COMMIT readback", + ], + remaining_obligations=[ + "all applicable G01-G16 cases and their required negative controls", + "an immutable manifest for the profile", + "fresh rebuild, exact resume and warm incremental output schedules distinguished", + "a non-vacuous P-023 compensated pair and raw C9 validation", + ], + fixture_originals=( + "publish-fixture synthesizes original bytes by re-encoding the fixture's parsed values" + " when no explicit originals are supplied; those rows are labelled synthesized on stderr" + " and validate_originals() is tautological for them. They are not engine originals."), + integrity_model=( + "self-certifying, not tamper-evident: a writer with access to the four math tables can" + " rewrite bytes, JSONB, both digests and the checkpoint coherently and is not detected." + " Digests bind an honest writer's own output; they are not an authentication scheme."), + pins_are_committed_sources=True, + sha256=source_pins, run_pins=run_pins) + (EVIDENCE / "s1-closure.json").write_text(json.dumps(closure, indent=2) + "\n") + audit = subprocess.run([sys.executable, str(ROOT / "delphi/tests/coordinator/audit_stages.py")], capture_output=True, text=True) + assert audit.returncode == 1, (audit.stdout, audit.stderr) + inventory = json.loads((EVIDENCE / "stage-inventory.json").read_text()) + assert inventory["reached"] == 25 and inventory["stage_inventory_gate"] == "PASS" + assert inventory["condition_states"]["O8"].startswith("PARTIAL"), audit.stderr + assert inventory["closed_conditions"] == [] and inventory["partial_conditions"] + assert len(inventory["open_conditions"]) == 8 and inventory["full_contract_gate"] == "FAIL" + summary = dict(run="P-026 step 4 / S1", python_full_suite=dict(passed=len(cases), failed=0, errors=0, skipped=0, + by_module=dict(counts), junit=str(junit.relative_to(ROOT))), rust_tests=rust_counts, + clippy="both feature sets: PASS --all-targets -D warnings", + builds=dict(release="PASS", debug_fault_injection="PASS", release_fault_injection="EXPECTED REFUSAL asserted in Python suite"), + stage_audit={k: inventory[k] for k in ("reached", "stage_inventory_gate", "full_contract_gate", "open_conditions", "condition_states", "closed_conditions", "partial_conditions")}, + replay_comparison=dict(checkpoints=len(replay["checkpoints"]), deltas=[len(c["deltas"]) for c in replay["checkpoints"]], + hashes_identical_to_previous_evidence=True, + observer_errors=len(replay["observer_errors"]), observations=replay["observations"]), + polarity="four artifacts and semantic tie control unchanged; compensated pairs equal, negative controls differ", + node_d4="0 differences; generation-zero route 200 / ETag 0 / conditional 304; no loadBundle claim", + scope="Rust versus Python rebuild-prefix campaign; Clojure remains the live production writer; no Rust/Clojure comparison", + compose_project=compose_project, postgres_port=run_pins["postgres_port"], uncommitted=False) + (EVIDENCE / "test-summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(dict(python=len(cases), rust=rust_counts, stages="25/25", + replay="/".join(str(len(c["deltas"])) for c in replay["checkpoints"]), + open_conditions=len(inventory["open_conditions"]), + O8=inventory["condition_states"]["O8"], full_contract_gate="FAIL"))) + + +if __name__ == "__main__": + main() diff --git a/delphi/polismath/engine_adapter.py b/delphi/polismath/engine_adapter.py new file mode 100644 index 0000000000..a2a58df084 --- /dev/null +++ b/delphi/polismath/engine_adapter.py @@ -0,0 +1,388 @@ +"""polis-engine/1 stdio worker, P-026 candidate profile (no persistence). + +The coordinator freezes files before dispatch; the worker admits them once. +Control stdout is JSONL only. Numerical routines and writer derivation are reused. +The local manifest/schedule schema is documented in coordinator-rs/README.md. +""" +import argparse +import contextlib +import hashlib +import json +import logging +import os +from pathlib import Path +import random +import sys + +PROTOCOL = "polis-engine/1" +CANDIDATE_SCHEMA = "polis-candidate-input/1" +ENGINE_VERSION = "python-conversation/p026-s1" +LIMIT = 65536 +BULK_LIMIT = 256 * 1024 * 1024 + + +class ProtocolError(Exception): + def __init__(self, code): + self.code = code + + +def fail(code="INVALID_INPUT"): + raise ProtocolError(code) + + +def strict_json(raw): + def pairs(items): + out = {} + for key, value in items: + if key in out: + fail() + out[key] = value + return out + try: + return json.loads(raw, object_pairs_hook=pairs, + parse_constant=lambda _: fail()) + except (ValueError, UnicodeError): + fail() + + +def integer(value, minimum=0): + if type(value) is not int or not minimum <= value <= 2**63 - 1: + fail() + return value + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def descriptor(root, path): + raw = (root / path).read_bytes() + return {"path": path, "bytes": len(raw), "sha256": sha(raw)} + + +def admitted(root, desc): + if not isinstance(desc, dict) or set(desc) != {"path", "bytes", "sha256"}: + fail() + if not isinstance(desc["path"], str) or not isinstance(desc["sha256"], str): + fail() + relative = Path(desc["path"]) + if relative.is_absolute() or ".." in relative.parts: + fail() + path = root + for part in relative.parts: + path = path / part + if path.is_symlink(): + fail() + if not path.is_file() or root.resolve() not in path.resolve().parents: + fail() + size = integer(desc["bytes"]) + if size > BULK_LIMIT or path.stat().st_size != size: + fail("RESOURCE_LIMIT") + raw = path.read_bytes() + if sha(raw) != desc["sha256"]: + fail("CHECKSUM_MISMATCH") + return raw + + +def emit_payloads(conv, zid, empty_contract=True): + from polismath.poller.math_writer import derive_bidtopid, derive_ptptstats + from polismath.utils.serialization import convert_numpy_types + main = conv.to_dict() + bid = derive_bidtopid(conv, zid) + stats = derive_ptptstats(conv, zid, main.get("user-vote-counts", {})) + if empty_contract and conv.raw_rating_mat.empty: + # Serialization-only correction; the engine's internal seed is unchanged. + main.update({"zid": zid, "n": 0, "n-cmts": 0, "tids": [], "in-conv": [], + "base-clusters": {k: [] for k in ("id", "members", "x", "y", "count")}, + "group-clusters": [], "group-votes": {}, "votes-base": {}, + "user-vote-counts": {}, "comment-priorities": {}, + "consensus": {"agree": [], "disagree": []}, "group-aware-consensus": {}, + "repness": {}, "meta-tids": sorted(conv.meta_tids), + "mod-in": sorted(conv.mod_in_tids), "mod-out": sorted(conv.mod_out_tids), + "lastVoteTimestamp": 0, "lastModTimestamp": conv.last_mod_timestamp, + "pca": {"center": [], "comps": [[], []], + "comment-projection": [[], []], "comment-extremity": []}}) + bid = {"zid": zid, "bidToPid": [], "lastVoteTimestamp": 0} + stats = {"zid": zid, "ptptstats": {}, "lastVoteTimestamp": 0} + return strict_json(json.dumps({"main": main, "bidtopid": bid, "ptptstats": stats}, + default=convert_numpy_types, allow_nan=False)) + + +class Adapter: + def __init__(self, input_root, output_root): + self.input_root = Path(input_root).resolve() + self.output_root = Path(output_root).resolve() + self.output_root.mkdir(parents=True, exist_ok=True) + self.state = "NEW" + self.identity = None + self.request_id = 0 + self.vote_cursor = self.mod_cursor = 0 + self.math_cursors = None + self.compute_id = None + self.operation = 0 + + def cursors(self): + return {"votes": {"slot": self.vote_cursor, + "sha256": sha(b"".join(self.vote_lines[:self.vote_cursor]))}, + "moderation": {"slot": self.mod_cursor, + "sha256": sha(b"".join(self.mod_lines[:self.mod_cursor]))}} + + def handle(self, req): + if set(req) != {"protocol", "run_id", "session_id", "request_id", "op", "payload"}: + fail() + if req["protocol"] != PROTOCOL: + fail("UNSUPPORTED_VERSION") + ident = (req["run_id"], req["session_id"]) + if not all(isinstance(x, str) and 0 < len(x) <= 128 for x in ident): + fail() + rid = integer(req["request_id"], 1) + if rid <= self.request_id or (self.identity is not None and ident != self.identity): + fail("INVALID_SEQUENCE") + self.identity, self.request_id = ident, rid + op, p = req["op"], req["payload"] + if not isinstance(p, dict): + fail() + if op == "initialize": + if self.state != "NEW": + fail("INVALID_SEQUENCE") + result = self.initialize(p) + self.state = "READY" + return result + if self.state != "READY": + fail("INVALID_SEQUENCE") + if self.operation >= len(self.schedule) or self.schedule[self.operation] != {"op": op, "payload": p}: + fail("INVALID_SEQUENCE") + self.operation += 1 + if op == "apply_votes": + self.apply_votes(p) + return {"observed_state_cursors": self.cursors()} + if op == "apply_moderation": + self.apply_mods(p) + return {"observed_state_cursors": self.cursors()} + if op == "compute": + if set(p) != {"compute_id", "logical_clock"}: + fail() + integer(p["logical_clock"]) + self.conv = self.conv.recompute() + self.compute_id = p["compute_id"] + self.math_cursors = self.cursors() + return {"compute_id": self.compute_id, "state_generation": rid, + "math_input_cursors": self.math_cursors} + if op == "snapshot": + return self.snapshot(p) + if op == "restore": + return self.restore(p) + if op == "close": + if p: + fail() + self.state = "CLOSED" + return {"closed": True} + fail("UNSUPPORTED_VERSION") + + def initialize(self, p): + from polismath.conversation.conversation import Conversation + import numpy as np + if set(p) != {"input_manifest", "resolved_schedule", "config", "required_capabilities", "admission"}: + fail() + if set(p["required_capabilities"]) - {"rebuild-prefix/1", "snapshot-moderation/1"}: + fail("UNSUPPORTED_VERSION") + admission = p["admission"] + keys = {"candidate_schema", "engine_version", "input_digest", "schedule_digest", "operation_id"} + if not isinstance(admission, dict) or set(admission) != keys or any( + not isinstance(v, str) or not 0 < len(v) <= 128 for v in admission.values()): + fail("MALFORMED_CANDIDATE") + if admission["candidate_schema"] != CANDIDATE_SCHEMA: + fail("CANDIDATE_SCHEMA_MISMATCH") + if admission["engine_version"] != ENGINE_VERSION: + fail("ENGINE_VERSION_MISMATCH") + manifest_raw = admitted(self.input_root, p["input_manifest"]) + if admission["input_digest"] != sha(manifest_raw): + fail("INPUT_DIGEST_MISMATCH") + schedule_raw = admitted(self.input_root, p["resolved_schedule"]) + if admission["schedule_digest"] != sha(schedule_raw): + fail("SCHEDULE_DIGEST_MISMATCH") + self.admission = admission.copy() + self.manifest = strict_json(manifest_raw) + manifest = self.manifest + if not isinstance(manifest, dict) or set(manifest) != {"schema", "fixture_id", "storage_agree_value", "ordering", "votes", "moderation", "parent"}: + fail() + if manifest["schema"] != CANDIDATE_SCHEMA: + fail("CANDIDATE_SCHEMA_MISMATCH") + self.zid = manifest["fixture_id"] + if not (type(self.zid) is int or isinstance(self.zid, str)): + fail() + self.sign = manifest["storage_agree_value"] + if type(self.sign) is not int or self.sign not in (-1, 1): + fail() + # Rev5 item 1: a live profile declares its own normalization, and the + # tie key must be over the semantic vote; a frozen replay names its + # pinned order instead. Nothing else is admitted. + order = manifest["ordering"] + if isinstance(order, dict): + if order.get("schema") != "polis-order/1" or not order.get("algorithm_digest"): + fail() + if order.get("semantic_vote") != "raw_vote * storage_agree_value": + fail() + if order.get("storage_agree_value") != self.sign: + fail() + elif not (type(order) is str and order): + fail() + self.vote_lines = admitted(self.input_root, manifest["votes"]).splitlines(keepends=True) + self.mod_lines = admitted(self.input_root, manifest["moderation"]).splitlines(keepends=True) + self.votes = [strict_json(line) for line in self.vote_lines] + self.mods = [strict_json(line) for line in self.mod_lines] + for i, (row, line) in enumerate(zip(self.votes, self.vote_lines), 1): + if not line.endswith(b"\n") or set(row) != {"slot", "source_ordinal", "stream_ordinal", "created_ms", "pid", "tid", "raw_vote", "weight_x_32767"}: + fail() + for key in ("slot", "source_ordinal", "stream_ordinal", "created_ms", "pid", "tid"): + integer(row[key]) + if row["slot"] != i or type(row["raw_vote"]) is not int or row["raw_vote"] not in (-1, 0, 1): + fail() + if row["weight_x_32767"] is not None: + integer(row["weight_x_32767"], -(2**63)) + for i, row in enumerate(self.mods, 1): + if set(row) != {"slot", "state"} or row["slot"] != i: + fail() + state = row["state"] + if set(state) != {"mod_out_tids", "mod_in_tids", "meta_tids", "mod_out_ptpts", "lastModTimestamp"}: + fail() + for key in ("mod_out_tids", "mod_in_tids", "meta_tids", "mod_out_ptpts"): + for v in state[key]: + integer(v) + schedule = strict_json(schedule_raw) + if set(schedule) != {"schema", "operations"} or schedule["schema"] != "polis-schedule/1": + fail("UNSUPPORTED_VERSION") + self.schedule = schedule["operations"] + cfg = p["config"] + if set(cfg) != {"profile", "seed", "pca_mode", "empty_contract", "init_vector"} or cfg["profile"] != "candidate-profile": + fail("UNSUPPORTED_VERSION") + if cfg["pca_mode"] not in ("powerit", "sklearn") or cfg["init_vector"] not in ("ones", "engine-default"): + fail("UNSUPPORTED_VERSION") + integer(cfg["seed"]) + if type(cfg["empty_contract"]) is not bool: + fail() + os.environ["POLISMATH_PCA_IMPL"] = cfg["pca_mode"] + random.seed(cfg["seed"]) + np.random.seed(cfg["seed"]) + self.config = cfg + self.conv = Conversation(self.zid, last_updated=1) + self.conv.last_updated = 0 + if cfg["init_vector"] == "ones": + self.conv.pca = {"center": np.zeros(1), "comps": np.array([[1.0], [1.0]])} + return {"engine": "python-conversation", "admission": self.admission, "profile": "candidate-profile", + "restore_profiles": ["rebuild-prefix/1"], + "reset_on_restore": ["raw_rating_mat", "rating_mat", "group_clusterings", "group_k_smoother"], + "observed_state_cursors": self.cursors()} + + def range(self, p, cursor, count): + if set(p) != {"from_slot_exclusive", "to_slot_inclusive"}: + fail() + start, end = integer(p["from_slot_exclusive"]), integer(p["to_slot_inclusive"]) + if start != cursor or not start < end <= count: + fail("INVALID_SEQUENCE") + return start, end + + def apply_votes(self, p): + start, end = self.range(p, self.vote_cursor, len(self.votes)) + rows = [{"pid": v["pid"], "tid": v["tid"], "created": v["created_ms"], + "vote": v["raw_vote"] * self.sign} for v in self.votes[start:end]] + self.conv = self.conv.update_votes({"votes": rows, + "lastVoteTimestamp": max([self.conv.last_updated] + [v["created"] for v in rows])}, recompute=False) + self.vote_cursor = end + + def apply_mods(self, p): + start, end = self.range(p, self.mod_cursor, len(self.mods)) + for row in self.mods[start:end]: + # update_moderation ignores empty collections; replace snapshot sets. + for key in ("mod_out_tids", "mod_in_tids", "meta_tids", "mod_out_ptpts"): + setattr(self.conv, key, set()) + self.conv = self.conv.update_moderation(row["state"], recompute=False) + self.mod_cursor = end + + def snapshot(self, p): + if set(p) != {"checkpoint_id"} or self.compute_id is None: + fail("INVALID_SEQUENCE") + name = p["checkpoint_id"] + if not isinstance(name, str) or not name or any(c not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_" for c in name): + fail() + dest = self.output_root / name + dest.mkdir() # duplicate checkpoint IDs are errors + payload = emit_payloads(self.conv, self.zid, self.config["empty_contract"]) + # Restore uses the uncorrected full engine state, not the semantic empty view. + from polismath.utils.serialization import convert_numpy_types + files = {} + for key, value in dict(payload, restore=self.conv.to_dict()).items(): + path = f"{name}/{key}.json" + raw = json.dumps(value, default=convert_numpy_types, allow_nan=False).encode() + (self.output_root / path).write_bytes(raw) + files[key] = descriptor(self.output_root, path) + manifest = {"schema": "polis-candidate-checkpoint/1", "protocol": PROTOCOL, + "run_id": self.identity[0], "session_id": self.identity[1], "fixture_id": self.zid, + "checkpoint_id": name, "compute_id": self.compute_id, "profile": "candidate-profile", + "admission": self.admission, "output_schema": "polis-candidate-math-output/1", "state_schema": "rebuild-prefix/1", + "persistence": False, "math_input_cursors": self.math_cursors, + "observed_state_cursors": self.cursors(), "files": files} + temp = dest / "manifest.tmp" + with temp.open("wb") as fh: + fh.write(json.dumps(manifest, allow_nan=False).encode()) + fh.flush() + os.fsync(fh.fileno()) + temp.rename(dest / "manifest.json") + return {"checkpoint_id": name, "manifest": descriptor(self.output_root, f"{name}/manifest.json")} + + def restore(self, p): + from polismath.conversation.conversation import Conversation + if self.vote_cursor or self.mod_cursor or self.compute_id is not None: + fail("INVALID_SEQUENCE") + if set(p) != {"profile", "parent", "payload", "vote_cursor", "moderation_cursor"} or p["profile"] != "rebuild-prefix/1": + fail("STATE_INCOMPATIBLE") + if p["parent"] != self.manifest["parent"] or not p["parent"]: + fail("STATE_INCOMPATIBLE") + blob = strict_json(admitted(self.input_root, p["payload"])) + if blob.get("zid") != self.zid: + fail("STATE_INCOMPATIBLE") + self.conv = Conversation.from_dict(blob) + self.conv.last_updated = 0 + if p["vote_cursor"]: + self.apply_votes({"from_slot_exclusive": 0, "to_slot_inclusive": p["vote_cursor"]}) + if p["moderation_cursor"]: + self.apply_mods({"from_slot_exclusive": 0, "to_slot_inclusive": p["moderation_cursor"]}) + return {"observed_state_cursors": self.cursors(), "profile": "rebuild-prefix/1"} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input-root", required=True) + parser.add_argument("--output-root", required=True) + args = parser.parse_args() + worker = Adapter(args.input_root, args.output_root) + logging.disable(logging.CRITICAL) + while True: + raw = sys.stdin.buffer.readline(LIMIT + 1) + if not raw: + return 0 if worker.state == "CLOSED" else 1 + req = {} + try: + if len(raw) > LIMIT or not raw.endswith(b"\n"): + fail("RESOURCE_LIMIT") + parsed = strict_json(raw) + if not isinstance(parsed, dict): + fail() + req = parsed + with contextlib.redirect_stdout(sys.stderr): + result = worker.handle(req) + response = {**{k: req[k] for k in ("protocol", "run_id", "session_id", "request_id")}, + "ok": True, "result": result} + except Exception as exc: + code = exc.code if isinstance(exc, ProtocolError) else "COMPUTE_FAILED" + response = {**{k: req.get(k) for k in ("protocol", "run_id", "session_id", "request_id")}, + "ok": False, "error": {"code": code, "request_id": req.get("request_id"), "retryable": False}} + print(json.dumps(response, allow_nan=False), flush=True) + return 1 + print(json.dumps(response, allow_nan=False), flush=True) + if worker.state == "CLOSED": + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/delphi/polismath/replay/certify.py b/delphi/polismath/replay/certify.py index fba6b610d7..e58b0fbcba 100644 --- a/delphi/polismath/replay/certify.py +++ b/delphi/polismath/replay/certify.py @@ -1190,6 +1190,7 @@ def _write_manifest(manifest_path: Path, manifest: dict[str, Any]) -> None: _ENGINE_TREE_EXCLUDE: tuple[str, ...] = ( "poller/", "replay/certify.py", + "replay/coordinator_driver.py", # P-045 bridge: harness-only, like certify.py "replay/poller_equiv.py", "replay/prodclone.py", "replay/shard_bench.py", diff --git a/delphi/polismath/replay/coordinator_driver.py b/delphi/polismath/replay/coordinator_driver.py new file mode 100644 index 0000000000..12752fa9ac --- /dev/null +++ b/delphi/polismath/replay/coordinator_driver.py @@ -0,0 +1,1505 @@ +"""P-045 bridge: a third certify PRODUCER (``rust``) beside ``clj`` and ``py``. + +``rust`` means the Rust coordinator + the pinned Python science worker + actual +Postgres publication (P-044/P-045). It does NOT mean Rust math, and it is never +the debug ``publish-fixture`` lane (that is ``store-fixture``). Each public +battery entry keeps explicit clj/py/rust producer rows and pairwise results; +creating a row is not evidence that it passes. + +This module is HARNESS-ONLY. It is listed in :data:`certify._ENGINE_TREE_EXCLUDE` +(slice 1), exactly like the four sibling harness files already there, so editing +it never re-keys the Python engine recording cache and never changes +``run_provenance.engine_tree_sha256`` for a legacy two-driver command. That is +the mechanism the engine hash was built for (see ``engine_tree_hash``'s +docstring), not a weakening of it: a Rust coordinator driver is not the Python +engine surface. + +DELIBERATELY ABSENT here (slice 3, BLOCKED on Colin's lifecycle decision and on +the closed ``p045-replay-plan/1`` schema + ordered-ingress DDL that P-045 §rev2 +records as undefined): + +* the ``once --replay-plan FILE --checkpoint INT --evidence-dir DIR`` forced + compute path and the ``replay`` warm session (P026_INCREMENTAL=0 does NOT force + a compute — ``process_owned`` still skips an unchanged fingerprint); +* the real Rust binary launch, real Postgres publication and the 20 fresh + reference repetitions. + +Until those land, :func:`RUST.record` returns an honest UNSUPPORTED_PROFILE / +INCONCLUSIVE receipt for every entry rather than a fabricated pass, and the Rust +producer is NEVER implemented by calling ``ensure_py_recording`` and relabelling. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Any, Callable, Optional, TypedDict + +from polismath.replay import certify as cert + +# --------------------------------------------------------------------------- +# Closed schema / identifier constants. +# --------------------------------------------------------------------------- +#: The SINGLE policy name used by BOTH P-044 and P-045. The previous ``p044`` +#: spelling is NOT a second accepted wire format. +POLICY_SCHEMA = "p045-policy/1" +DRIVER_CAPABILITIES_SCHEMA = "p045-driver-capabilities/1" +STAGE_BINDING_SCHEMA = "p045-stage-binding/1" +STAGE_CONTEXT_SCHEMA = "p045-stage-context/1" +#: New report schema for the three-producer run; ``/1`` is preserved untouched +#: for legacy two-driver invocations (see certify.run_battery). +CERTIFICATION_RUN_SCHEMA_V2 = "polis-certification-run/2" +#: Bridge recording-cache identity version. Bumping it invalidates only bridge +#: recordings, never the legacy clj/py recording caches. +BRIDGE_CACHE_SCHEMA = "p045-bridge-cache/1" + +#: Producer IDs are EXACTLY these three. Unknown/repeated/empty values fail +#: configuration; Rust certification requires both references (clj AND py). +DRIVER_IDS: tuple[str, ...] = ("clj", "py", "rust") +LEGACY_DRIVER_IDS: tuple[str, ...] = ("clj", "py") + +#: Campaign profiles. ``battery-chain/1`` needs the warm-worker lifecycle that +#: S1 does not advertise (slice 3); ``snapshot-rebuild/1`` is the rebuild-each-cut +#: profile S1 actually implements. Neither is a production default. +PROFILE_BATTERY_CHAIN = "battery-chain/1" +PROFILE_SNAPSHOT_REBUILD = "snapshot-rebuild/1" +PROFILES: tuple[str, ...] = (PROFILE_BATTERY_CHAIN, PROFILE_SNAPSHOT_REBUILD) + +#: S1's closed admission fields (engine_adapter.CANDIDATE_SCHEMA / ENGINE_VERSION). +#: A changed worker/profile must NEGOTIATE a new version, not claim these bytes +#: implement a capability S1 never advertised. +S1_CANDIDATE_SCHEMA = "polis-candidate-input/1" +S1_ENGINE_VERSION = "python-conversation/p026-s1" +S1_ADMISSION_FIELDS: tuple[str, ...] = ( + "candidate_schema", "engine_version", "input_digest", "schedule_digest", + "operation_id", +) + + +class ProducerStatus(str, Enum): + """Row-level status. Creating a row is never a pass; a missing capability is + UNSUPPORTED_PROFILE and an absent-but-buildable result is INCONCLUSIVE.""" + + PASS = "PASS" + FAIL = "FAIL" + INCONCLUSIVE = "INCONCLUSIVE" + UNSUPPORTED_PROFILE = "UNSUPPORTED_PROFILE" + + +class BridgeError(RuntimeError): + """A bridge configuration/contract failure. Carries a stable ``stage``.""" + + def __init__(self, stage: str, message: str) -> None: + super().__init__(message) + self.stage = stage + + +# --------------------------------------------------------------------------- +# Driver registry. +# --------------------------------------------------------------------------- +@dataclass(frozen=True) +class DriverReceipt: + """What one producer's ``record`` actually produced for one entry.""" + + driver: str + loader: str + profile: str + status: ProducerStatus + reason: str + required_cuts: int + observed_cuts: int + out_dir: Optional[str] = None + evidence_sha256: Optional[str] = None + cache_provenance: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "driver": self.driver, "loader": self.loader, "profile": self.profile, + "status": self.status.value, "reason": self.reason, + "required_cuts": self.required_cuts, "observed_cuts": self.observed_cuts, + "out_dir": self.out_dir, "evidence_sha256": self.evidence_sha256, + "cache_provenance": self.cache_provenance, + } + + +# record(plan, entry, out_dir) -> DriverReceipt. Plan/entry are kept ``Any`` so +# the registry does not depend on the (slice-3, undefined) p045-replay-plan/1 +# object shape; the concrete Rust launcher binds them once that schema lands. +RecordFn = Callable[[Any, Any, Path], DriverReceipt] + + +@dataclass(frozen=True) +class DriverSpec: + """id, loader, executable/implementation digest, advertised capabilities and + the record function. ``loader`` is the recording SHAPE on disk; it is a + distinct axis from producer identity so a producer is never certified by + borrowing another's loader (the brief's "do not relabel Rust as py").""" + + id: str + loader: str + implementation_digest: str + capabilities: tuple[str, ...] + record: RecordFn + + def supports(self, profile: str) -> bool: + return profile in self.capabilities + + +def _plan_profile(plan: Any) -> str: + """The selected profile, whether ``plan`` is the (dict) bridge plan or an + object with a ``.profile`` attribute. The runner passes a dict, so reading it + only as an attribute would leave every receipt profile empty beneath a row + that says snapshot-rebuild/1.""" + if isinstance(plan, dict): + return plan.get("profile") or "" + return getattr(plan, "profile", "") or "" + + +def _legacy_receipt(driver: str, loader: str, profile: str, entry: Any) -> DriverReceipt: + """clj/py are the pinned references. Their real recording still runs through + :func:`certify.ensure_clj_recording`/:func:`certify.ensure_py_recording` + (unchanged, byte-for-byte); the registry only NAMES them as producers so the + three-producer inventory and pairing can address them uniformly. Without a + resolved plan (slice 3) this returns an INCONCLUSIVE placeholder rather than + launching an engine.""" + required = _required_cuts(entry) + return DriverReceipt( + driver=driver, loader=loader, profile=profile, + status=ProducerStatus.INCONCLUSIVE, + reason="reference recording not yet run in this bridge invocation", + required_cuts=required, observed_cuts=0, + ) + + +def _rust_record(plan: Any, entry: Any, out_dir: Path) -> DriverReceipt: + """The Rust producer. It NEVER borrows ``ensure_py_recording``. + + Without the closed ``p045-replay-plan/1`` schema and the plan-gated + ``once --replay-plan`` forced-compute path (slice 3, BLOCKED), there is no way + to require a real publication per scheduled cut, so every entry records + UNSUPPORTED_PROFILE with the reason named. This is the honest non-passing row + the brief mandates, not a fabricated pass.""" + required = _required_cuts(entry) + profile = _plan_profile(plan) + reason = ( + "rust producer requires the slice-3 p045-replay-plan/1 schema and the " + "plan-gated once --replay-plan forced-compute path, both BLOCKED on " + "Colin's lifecycle decision; P026_INCREMENTAL=0 alone does not force a " + "compute" + ) + return DriverReceipt( + driver="rust", loader="rust", profile=profile, + status=ProducerStatus.UNSUPPORTED_PROFILE, reason=reason, + required_cuts=required, observed_cuts=0, out_dir=str(out_dir), + ) + + +def _required_cuts(entry: Any) -> int: + for attr in ("checkpoints",): + val = getattr(entry, attr, None) + if val is not None: + try: + return len(val) + except TypeError: + pass + if isinstance(entry, dict) and isinstance(entry.get("checkpoints"), list): + return len(entry["checkpoints"]) + return 0 + + +CLJ = DriverSpec( + id="clj", loader="clj", implementation_digest="", + capabilities=(PROFILE_BATTERY_CHAIN, PROFILE_SNAPSHOT_REBUILD), + record=lambda plan, entry, out_dir: _legacy_receipt("clj", "clj", _plan_profile(plan), entry), +) +PY = DriverSpec( + id="py", loader="py", implementation_digest="", + capabilities=(PROFILE_BATTERY_CHAIN, PROFILE_SNAPSHOT_REBUILD), + record=lambda plan, entry, out_dir: _legacy_receipt("py", "py", _plan_profile(plan), entry), +) +#: Rust advertises NO campaign profile today: its capabilities are empty until a +#: reviewed build + capabilities --json reply pins them (slice 2/3). An empty +#: capability set yields UNSUPPORTED_PROFILE, never a speculative engine run. +RUST = DriverSpec( + id="rust", loader="rust", implementation_digest="", + capabilities=(), record=_rust_record, +) + +REGISTRY: dict[str, DriverSpec] = {"clj": CLJ, "py": PY, "rust": RUST} + + +def parse_drivers(value: str) -> tuple[str, ...]: + """``--drivers clj,py,rust`` -> validated, order-preserving, unique tuple. + Unknown/repeated/empty tokens fail configuration; rust requires both refs.""" + tokens = [t for t in value.split(",")] + if any(t == "" for t in tokens) or not tokens: + raise BridgeError("drivers", f"empty driver id in {value!r}") + seen: set[str] = set() + out: list[str] = [] + for t in tokens: + if t not in REGISTRY: + raise BridgeError("drivers", f"unknown driver id {t!r} (allowed: {DRIVER_IDS})") + if t in seen: + raise BridgeError("drivers", f"repeated driver id {t!r}") + seen.add(t) + out.append(t) + if "rust" in seen and not {"clj", "py"} <= seen: + raise BridgeError("drivers", "rust certification requires both clj and py references") + return tuple(out) + + +# --------------------------------------------------------------------------- +# Reviewed policy (p045-policy/1). +# --------------------------------------------------------------------------- +_HEX = frozenset("0123456789abcdef") + + +def _is_hex64(value: Any) -> bool: + return isinstance(value, str) and len(value) == 64 and set(value) <= _HEX + + +#: The complete closed key set of a p045-policy/1 object. Unknown keys are +#: rejected — a "closed" schema that silently ignores extra fields is not closed. +_POLICY_KEYS: frozenset[str] = frozenset({ + "schema", "profile", "field_policy_sha256", "approvals", + "replacement_assertions", "required_controls", "clock", +}) + + +@dataclass(frozen=True) +class Policy: + schema: str + profile: str + field_policy_sha256: str + approvals: tuple[str, ...] + replacement_assertions: tuple[str, ...] + required_controls: tuple[str, ...] + clock: dict[str, Any] + raw_sha256: str + #: The p045-policy/1 BODY is still undefined (slice 3, BLOCKED). This loader + #: validates shape only; it NEVER certifies a run. ``admitting`` stays False + #: until the reviewed policy schema exists, so nothing can mistake a + #: shape-valid file for slice-3 admission. + admitting: bool = False + + def to_dict(self) -> dict[str, Any]: + return { + "schema": self.schema, "profile": self.profile, + "field_policy_sha256": self.field_policy_sha256, + "approvals": list(self.approvals), + "replacement_assertions": list(self.replacement_assertions), + "required_controls": list(self.required_controls), + "clock": self.clock, "raw_sha256": self.raw_sha256, + "admitting": self.admitting, + } + + +def load_policy(path: str | Path) -> Policy: + """Load and shape-validate a ``p045-policy/1`` file as NON-ADMITTING + scaffolding (the policy body is still undefined — slice 3, BLOCKED). It is + NEVER derived from observed errors; a wrong schema/profile, a missing or + mistyped field, a non-hex digest OR an unknown key is a hard configuration + failure. The returned Policy has ``admitting == False``: shape-valid is not + slice-3 admission.""" + p = Path(path) + raw = p.read_bytes() + try: + obj = json.loads(raw) + except json.JSONDecodeError as exc: + raise BridgeError("policy", f"policy is not valid JSON: {exc}") from exc + if not isinstance(obj, dict): + raise BridgeError("policy", "policy must be a JSON object") + unknown = set(obj) - _POLICY_KEYS + if unknown: + raise BridgeError("policy", f"policy has unknown key(s): {sorted(unknown)}") + if obj.get("schema") != POLICY_SCHEMA: + raise BridgeError("policy", f"policy schema must be {POLICY_SCHEMA!r}, got {obj.get('schema')!r}") + profile = obj.get("profile") + if profile not in PROFILES: + raise BridgeError("policy", f"policy profile must be one of {PROFILES}, got {profile!r}") + required = ("field_policy_sha256", "approvals", "replacement_assertions", + "required_controls", "clock") + missing = [k for k in required if k not in obj] + if missing: + raise BridgeError("policy", f"policy missing required field(s): {missing}") + for list_field in ("approvals", "replacement_assertions", "required_controls"): + if not isinstance(obj[list_field], list) or not all(isinstance(x, str) for x in obj[list_field]): + raise BridgeError("policy", f"policy.{list_field} must be a list of strings") + if not isinstance(obj["clock"], dict): + raise BridgeError("policy", "policy.clock must be an object") + if not _is_hex64(obj["field_policy_sha256"]): + raise BridgeError("policy", "policy.field_policy_sha256 must be a 64-char lowercase hex sha256") + return Policy( + schema=POLICY_SCHEMA, profile=profile, + field_policy_sha256=obj["field_policy_sha256"], + approvals=tuple(obj["approvals"]), + replacement_assertions=tuple(obj["replacement_assertions"]), + required_controls=tuple(obj["required_controls"]), + clock=obj["clock"], + raw_sha256=hashlib.sha256(raw).hexdigest(), + admitting=False, + ) + + +# --------------------------------------------------------------------------- +# Bridge recording-cache key. +# --------------------------------------------------------------------------- +class BridgeCacheInputs(TypedDict, total=False): + driver_id: str + loader_id: str + binary_tree_sha256: str + worker_tree_sha256: str + votes_sha256: str + comments_sha256: Optional[str] + moderation_sha256: Optional[str] + convention: str + schedule_sha256: str + profile: str + initialization_schema: str + restore_schema: str + migration_sha256: str + policy_sha256: str + comparer_cfg_sha256: str + stage_binding_version: str + serialization_profile: str + + +def bridge_cache_key(inputs: BridgeCacheInputs) -> dict[str, Any]: + """Compose the bridge recording-cache identity. + + It binds driver AND loader IDs, the binary and worker tree hashes, all + input/comment/moderation hashes, convention, resolved schedule and profile, + initialization and restore schema, migrations, policy/comparer hashes, and + the stage-capture/serialization profile — exactly the fields the brief §2 + lists. Returns ``{"schema", "key", "bound": {...}}``; ``key`` is a sha256 over + the canonicalised bound fields. + + The bridge cache is SEPARATE from the legacy clj/py caches: it never shares + their namespace, so a bridge run cannot invalidate or be satisfied by a + two-driver recording.""" + bound = { + "driver_id": inputs["driver_id"], + "loader_id": inputs["loader_id"], + "binary_tree_sha256": inputs["binary_tree_sha256"], + "worker_tree_sha256": inputs["worker_tree_sha256"], + "votes_sha256": inputs["votes_sha256"], + "comments_sha256": inputs.get("comments_sha256"), + "moderation_sha256": inputs.get("moderation_sha256"), + "convention": inputs["convention"], + "schedule_sha256": inputs["schedule_sha256"], + "profile": inputs["profile"], + "initialization_schema": inputs["initialization_schema"], + "restore_schema": inputs["restore_schema"], + "migration_sha256": inputs["migration_sha256"], + "policy_sha256": inputs["policy_sha256"], + "comparer_cfg_sha256": inputs["comparer_cfg_sha256"], + "stage_binding_version": inputs.get("stage_binding_version", STAGE_BINDING_SCHEMA), + "serialization_profile": inputs["serialization_profile"], + "cache_schema": BRIDGE_CACHE_SCHEMA, + } + canon = json.dumps(bound, sort_keys=True, separators=(",", ":")).encode() + return {"schema": BRIDGE_CACHE_SCHEMA, "key": hashlib.sha256(canon).hexdigest(), "bound": bound} + + +# --------------------------------------------------------------------------- +# Three-producer inventory rows (never hashed; purely additive). +# --------------------------------------------------------------------------- +def three_producer_inventory(expected: cert.ExpectedEntry, profile: str, + drivers: tuple[str, ...] = DRIVER_IDS) -> list[dict[str, Any]]: + """Per-entry inventory rows for the SELECTED ``drivers`` on ``profile``. + + With the default clj/py/rust the legacy two rows are byte-identical to + ``ExpectedEntry.inventory()`` and the rust row is appended; a narrower + selection emits exactly those producers (so the report never claims to have + invoked a driver the invocation did not select). A driver that does not + advertise ``profile`` is marked UNSUPPORTED_PROFILE up front.""" + if profile not in PROFILES: + raise BridgeError("profile", f"unknown profile {profile!r} (allowed: {PROFILES})") + rows: list[dict[str, Any]] = [] + for engine in drivers: + spec = REGISTRY[engine] + rows.append({ + "dataset": expected.entry.dataset, + "schedule_id": expected.entry.schedule_id, + "role": expected.entry.role or f"{expected.entry.dataset}:{expected.entry.schedule_id}", + "engine": engine, + "loader": spec.loader, + "profile": profile, + "coverage": expected.spec.coverage, + "stream_end": expected.stream_end, + "checkpoints": expected.checkpoints, + "advertised": spec.supports(profile), + "status": (ProducerStatus.INCONCLUSIVE if spec.supports(profile) + else ProducerStatus.UNSUPPORTED_PROFILE).value, + }) + return rows + + +# =========================================================================== +# P-045 slice 2 (identity / readback / observer / stage sibling). +# +# These grade a `rust` producer on the ONLY profile the binary implements today, +# snapshot-rebuild/1 — the scoped rebuild verdict both P-044 and P-045 refuse to +# call a battery-chain PASS. They are PURE validators over readback/checkpoint +# dicts, so they run with neither Postgres nor the binary; the real DB readback, +# the live observer and the actual worker launch are slice 3 (BLOCKED). The +# once --replay-plan/--checkpoint/--evidence-dir forced-compute path is NOT +# touched here and stays as it is on the S1 binary. +# =========================================================================== + +#: S1's closed candidate-checkpoint wire values (engine_adapter.snapshot). The +#: bridge BINDS these, not just key presence: the adapter emits exactly this +#: versioned identity, and a changed worker must NEGOTIATE a new version. +S1_CHECKPOINT_SCHEMA = "polis-candidate-checkpoint/1" +#: The STORE's persisted checkpoint envelope (Rust ResultsStore, store.rs) — a +#: DIFFERENT object from the worker's polis-candidate-checkpoint/1. +STORE_CHECKPOINT_SCHEMA = "polis-coordinator/1" +S1_PROTOCOL = "polis-engine/1" +S1_OUTPUT_SCHEMA = "polis-candidate-math-output/1" +S1_STATE_SCHEMA = "rebuild-prefix/1" +S1_PROFILE_WIRE = "candidate-profile" + +#: Retained checkpoint fields (brief §5), each with the concrete check applied. +#: "str+" = nonempty string; "int" = int and not bool; "bool" = bool; a literal +#: string = that exact value. +S1_CHECKPOINT_FIELDS: tuple[str, ...] = ( + "protocol", "fixture_id", "run_id", "session_id", "compute_id", + "checkpoint_id", "output_schema", "state_schema", "profile", "persistence", +) +_S1_CHECKPOINT_EXPECT: dict[str, Any] = { + "protocol": S1_PROTOCOL, "fixture_id": "int", "run_id": "str+", + "session_id": "str+", "compute_id": "str+", "checkpoint_id": "str+", + "output_schema": S1_OUTPUT_SCHEMA, "state_schema": S1_STATE_SCHEMA, + "profile": S1_PROFILE_WIRE, "persistence": "false", +} + +#: Identity fields the caller may bind expected values for (per-run, not global). +#: fixture_id is bound too (a checkpoint for fixture 999 must not pass when the +#: caller expects fixture 1). +S1_EXPECTED_IDENTITY_FIELDS = ("fixture_id", "run_id", "session_id", "compute_id", "checkpoint_id") +S1_EXPECTED_ADMISSION_FIELDS = ("input_digest", "schedule_digest", "operation_id") + +#: The COMPLETE closed key set of the worker's polis-candidate-checkpoint/1 +#: envelope (engine_adapter.snapshot). All 15 are mandatory; unknown top-level +#: keys are rejected. +S1_CHECKPOINT_CLOSED_KEYS = frozenset({ + "schema", "protocol", "run_id", "session_id", "fixture_id", "checkpoint_id", + "compute_id", "profile", "admission", "output_schema", "state_schema", + "persistence", "math_input_cursors", "observed_state_cursors", "files", +}) +#: The four output files the worker snapshots (emit_payloads + restore), each a +#: {path, bytes, sha256} descriptor. +S1_FILE_KEYS = ("main", "bidtopid", "ptptstats", "restore") +_CURSOR_STREAMS = ("votes", "moderation") + + +def _nonempty_str(v: Any) -> bool: + return isinstance(v, str) and len(v) > 0 + + +def _plain_int(v: Any) -> bool: + return isinstance(v, int) and not isinstance(v, bool) + + +# --------------------------------------------------------------------------- +# ONE typed admission schema, and one total admitter derived from it. +# +# Every bundle/checkpoint/envelope structure is described declaratively (kind, +# rank, element kind, nullability, closedness); :func:`_admit` walks any value +# against a spec and returns GRADED failures — it never raises on a wrong type, +# so admission is total and there are no ungraded TypeErrors. Relational and +# custody checks (partition, positional equality, count==len, group references, +# value bindings) run AFTER admission, on already-typed data. +# --------------------------------------------------------------------------- +Spec = dict # {"kind": ..., ...} + + +def _admit(value: Any, spec: Spec, path: str = "") -> list[str]: + here = path or "" + + def bad(default: str) -> list[str]: + return [f"{here}: {spec.get('msg', default)}"] + + kind = spec["kind"] + if kind == "any": + return [] + if kind == "int": + if not _plain_int(value): + return bad(f"must be a (non-boolean) integer, got {value!r}") + if "min" in spec and value < spec["min"]: + return bad(f"must be >= {spec['min']}, got {value!r}") + return [] + if kind == "bool": + return [] if isinstance(value, bool) else bad(f"must be a boolean, got {value!r}") + if kind == "str": + if not isinstance(value, str): + return bad(f"must be a string, got {value!r}") + if spec.get("nonempty") and not value: + return bad("must be a nonempty string") + return [] + if kind == "hex64": + return [] if _is_hex64(value) else bad(f"must be a 64-char lowercase hex sha256, got {value!r}") + if kind == "const": + want = spec["value"] + ok = value == want and type(value) is type(want) + return [] if ok else bad(f"must be {want!r}, got {value!r}") + if kind == "list": + if not isinstance(value, list): + return bad(f"must be a list, got {value!r}") + if "min_len" in spec and len(value) < spec["min_len"]: + return bad(f"must have at least {spec['min_len']} element(s)") + out: list[str] = [] + for i, el in enumerate(value): + out.extend(_admit(el, spec["of"], f"{path}[{i}]")) + return out + if kind == "object": + if not isinstance(value, dict): + return bad(f"must be an object, got {value!r}") + out = [] + fields: dict[str, Spec] = spec.get("fields", {}) + optional = set(spec.get("optional", ())) + if spec.get("closed"): + unknown = sorted(set(value) - set(fields)) + if unknown: + out.append(f"{here}: unknown key(s) {unknown}") + for k, sub in fields.items(): + if k not in value: + if k not in optional: + out.append(f"{here}: missing {k}") + else: + out.extend(_admit(value[k], sub, f"{path}.{k}" if path else k)) + return out + return bad(f"unknown schema kind {kind!r}") + + +# --- reusable leaf specs --- +_HEXSPEC = {"kind": "hex64"} +_STR = {"kind": "str", "nonempty": True} +_NNINT = {"kind": "int", "min": 0} +_PID = {"kind": "int"} # a participant id: integer, never bool/float +_INT_LIST = {"kind": "list", "of": _PID} + +# --- worker envelope (polis-candidate-checkpoint/1) --- +_CURSOR = {"kind": "object", "closed": True, "fields": {"slot": _NNINT, "sha256": _HEXSPEC}} +_CURSOR_MAP = {"kind": "object", "closed": True, + "fields": {"votes": _CURSOR, "moderation": _CURSOR}} +_FILE_DESC = {"kind": "object", "closed": True, + "fields": {"path": _STR, "bytes": _NNINT, "sha256": _HEXSPEC}} +_FILES = {"kind": "object", "closed": True, "fields": {k: _FILE_DESC for k in S1_FILE_KEYS}} +_ADMISSION_SPEC = {"kind": "object", "closed": True, "fields": { + "candidate_schema": {"kind": "const", "value": S1_CANDIDATE_SCHEMA}, + "engine_version": {"kind": "const", "value": S1_ENGINE_VERSION, + "msg": f"engine_version must be {S1_ENGINE_VERSION!r} " + f"(negotiate a new version rather than claim S1 bytes)"}, + "input_digest": _STR, "schedule_digest": _STR, "operation_id": _STR}} +_S1_ENVELOPE_SPEC = {"kind": "object", "closed": True, "fields": { + "schema": {"kind": "const", "value": S1_CHECKPOINT_SCHEMA}, + "protocol": {"kind": "const", "value": S1_PROTOCOL}, + "run_id": _STR, "session_id": _STR, "compute_id": _STR, "checkpoint_id": _STR, + "fixture_id": _NNINT, "profile": {"kind": "const", "value": S1_PROFILE_WIRE}, + "admission": _ADMISSION_SPEC, + "output_schema": {"kind": "const", "value": S1_OUTPUT_SCHEMA}, + "state_schema": {"kind": "const", "value": S1_STATE_SCHEMA}, + "persistence": {"kind": "const", "value": False}, + "math_input_cursors": _CURSOR_MAP, "observed_state_cursors": _CURSOR_MAP, + "files": _FILES}} + +# --- observer payloads --- +_BASE_CLUSTERS_SPEC = {"kind": "object", "closed": False, "optional": ("count",), "fields": { + "id": {"kind": "list", "of": _PID}, + "members": {"kind": "list", "of": _INT_LIST}, + "count": {"kind": "list", "of": _PID}}} +_GROUP_SPEC = {"kind": "list", "of": {"kind": "object", "closed": False, + "fields": {"members": {"kind": "list", "of": _PID}}}} +_MAIN_SPEC = {"kind": "object", "closed": False, "optional": ("group-clusters",), + "fields": {"base-clusters": _BASE_CLUSTERS_SPEC, "group-clusters": _GROUP_SPEC}} +_BIDTOPID_SPEC = {"kind": "object", "closed": True, "fields": { + "zid": {"kind": "any"}, "lastVoteTimestamp": {"kind": "any"}, + "bidToPid": {"kind": "list", "of": _INT_LIST}}} + +# --- store checkpoint (polis-coordinator/1) --- +# engine.rs:374,393-396 persists {slot, sha256} for BOTH streams; the store +# checkpoint reuses the same typed cursor as the worker envelope (_CURSOR). +_STORE_CURSOR_MAP = {"kind": "object", "closed": True, + "fields": {"votes": _CURSOR, "moderation": _CURSOR}} +_STORE_CHECKPOINT_SPEC = {"kind": "object", "closed": False, "fields": { + "schema": {"kind": "const", "value": STORE_CHECKPOINT_SCHEMA}, + "operation_id": _STR, "publisher_epoch": _NNINT, + "original_digests": {"kind": "object", "closed": False, "fields": {}}, + "payload_digests": {"kind": "object", "closed": False, "fields": {}}, + "cursors": _STORE_CURSOR_MAP}} + + +def validate_s1_identity(manifest: dict[str, Any], *, + expected_admission: Optional[dict[str, Any]] = None, + expected_identity: Optional[dict[str, Any]] = None, + expected_cursors: Optional[dict[str, Any]] = None, + expected_files: Optional[dict[str, Any]] = None) -> list[str]: + """Check S1's closed, VERSIONED identity on a candidate checkpoint manifest. + + Binds the checkpoint schema (``polis-candidate-checkpoint/1``); requires the + admission block to hold EXACTLY the five fields (no unknown keys) with + candidate_schema/engine_version at the pinned S1 constants and + input_digest/schedule_digest/operation_id NONEMPTY strings; and binds each + retained checkpoint field to its expected value/type — nonempty strings, + non-boolean fixture_id, ``persistence`` EXACTLY False (pinned engine.rs + requires false), and the pinned output/state/profile wire values. + + Per-run EXPECTED identity is bound when supplied: ``expected_admission`` + (``input_digest``/``schedule_digest``/``operation_id`` expected values) and + ``expected_identity`` (``run_id``/``session_id``/``compute_id``/ + ``checkpoint_id`` expected values). Without them a manifest is validated for + the closed ENVELOPE only; a real acceptance MUST supply the operation + identity so a well-typed FOREIGN session/digest is rejected, not accepted. + A different engine_version is never accepted as "implements a capability S1 + never advertised". Returns failures (empty == identity intact).""" + # Total admission against the ONE worker-envelope schema (no per-case checks, + # no ungraded exceptions): types, ranks, closedness, nonnegative slots/lengths + # and hex digests all come from _S1_ENVELOPE_SPEC. + fails: list[str] = _admit(manifest, _S1_ENVELOPE_SPEC) + if not isinstance(manifest, dict): + return fails # a non-object envelope is graded; do not dereference it + admission = manifest.get("admission") if isinstance(manifest.get("admission"), dict) else {} + + # Cursor custody (value binding beyond shape): at the snapshot seam both maps + # bind to the same independently derived source cursors, so they must agree + # with each other and with any supplied expected cursors. + mic, osc = manifest.get("math_input_cursors"), manifest.get("observed_state_cursors") + if isinstance(mic, dict) and isinstance(osc, dict) and mic != osc: + fails.append("math_input_cursors and observed_state_cursors disagree " + "(both must bind the same source cursors at the snapshot seam)") + if expected_cursors is not None: + for name, cur in (("math_input_cursors", mic), ("observed_state_cursors", osc)): + if not _json_type_equal(cur, expected_cursors): + fails.append(f"{name} does not equal the expected source cursors") + if expected_files is not None and not _json_type_equal(manifest.get("files"), expected_files): + fails.append("files do not equal the expected output-file descriptors") + + # Per-run expected-identity binding (rejects a well-typed FOREIGN identity). + if expected_admission is not None: + for f in S1_EXPECTED_ADMISSION_FIELDS: + if f in expected_admission and admission.get(f) != expected_admission[f]: + fails.append(f"admission.{f} {admission.get(f)!r} != expected {expected_admission[f]!r}") + if expected_identity is not None: + for f in S1_EXPECTED_IDENTITY_FIELDS: + if f in expected_identity and manifest.get(f) != expected_identity[f]: + fails.append(f"checkpoint.{f} {manifest.get(f)!r} != expected {expected_identity[f]!r}") + return fails + + +def bind_campaign_cut(*, compute_id: str, checkpoint_id: str, global_cut_index: int, + schedule_digest: str, campaign_plan_sha256: str) -> dict[str, Any]: + """The campaign sidecar entry binding S1's per-session identity to ONE global + cut. S1 resets names to compute-0/checkpoint-0 in a fresh session, so local + name reuse across different sessions is legal; this record makes the mapping + explicit and distinguishes the campaign plan hash from S1's schedule_digest.""" + return { + "compute_id": compute_id, "checkpoint_id": checkpoint_id, + "global_cut_index": int(global_cut_index), + "s1_schedule_digest": schedule_digest, + "campaign_plan_sha256": campaign_plan_sha256, + } + + +# --------------------------------------------------------------------------- +# Four-row publication readback (main/bidtopid/ptptstats/ticks). +# --------------------------------------------------------------------------- +class TableRow(TypedDict, total=False): + math_tick: int + caching_tick: Optional[int] + data: dict # parsed JSONB — correspondence ONLY, never the origin + original_bytes: str # the exact bytes the worker emitted (origin of truth) + original_sha256: str + + +class TicksRow(TypedDict, total=False): + math_tick: int + caching_tick: Optional[int] + publisher_epoch: int + operation_id: str + original_digests: dict # {"main","bidtopid","ptptstats"} + + +class ReadbackBundle(TypedDict, total=False): + zid: int + math_env: str + main: TableRow + bidtopid: TableRow + ptptstats: TableRow + ticks: TicksRow + + +_COMPANIONS = ("bidtopid", "ptptstats") +_PAYLOAD_TABLES = ("main", "bidtopid", "ptptstats") + + +def _row_data(bundle: Any, name: str) -> Any: + """The ``data`` of a bundle row, or None when the row is absent or not an + object — so a non-dict row never crashes a public accessor.""" + row = bundle.get(name) if isinstance(bundle, dict) else None + return row.get("data") if isinstance(row, dict) else None + + +def _dict_or_empty(value: Any) -> dict: + """``value`` when it is a dict, else ``{}`` — a graded-safe accessor so a + malformed (non-object) map is compared/looked-up without raising.""" + return value if isinstance(value, dict) else {} + + +def _json_type_equal(a: Any, b: Any) -> bool: + """Type-aware JSON equality: PostgreSQL is allowed to normalize NUMBER + spelling (1 vs 1.0), but a JSON boolean is NEVER equal to an integer — + Python's ``1 == True`` must not launder a bool into a faithful integer + payload. Recurses structurally; dict key sets and list lengths must match.""" + if isinstance(a, bool) or isinstance(b, bool): + return isinstance(a, bool) and isinstance(b, bool) and a == b + if isinstance(a, (int, float)) and isinstance(b, (int, float)): + return a == b + if isinstance(a, dict) and isinstance(b, dict): + return a.keys() == b.keys() and all(_json_type_equal(a[k], b[k]) for k in a) + if isinstance(a, list) and isinstance(b, list): + return len(a) == len(b) and all(_json_type_equal(x, y) for x, y in zip(a, b)) + return type(a) is type(b) and a == b + + +def _serde_json_string(s: str) -> bytes: + """serde_json's string encoding (``serde_json::to_writer`` for a &str): the + two-char escapes for " \\ \\b \\f \\n \\r \\t, ``\\u00XX`` for the remaining + C0 controls, ``/`` NOT escaped, and every other codepoint written through as + UTF-8 (no ``\\uXXXX`` for non-ASCII — the store writes Unicode through serde). + + An UNPAIRED surrogate (U+D800..U+DFFF) cannot exist in a Rust ``String``, so a + JSON text carrying one fails to parse into ``serde_json::Value`` on the Rust + side; mirror that as a graded refusal (BridgeError) rather than a + UnicodeEncodeError. A valid non-BMP scalar (a proper surrogate pair, which + Python's JSON parser has already combined into a single codepoint) encodes + through as UTF-8 exactly as serde does.""" + out = bytearray(b'"') + _short = {'"': b'\\"', '\\': b'\\\\', '\b': b'\\b', '\f': b'\\f', + '\n': b'\\n', '\r': b'\\r', '\t': b'\\t'} + for ch in s: + code = ord(ch) + esc = _short.get(ch) + if esc is not None: + out += esc + elif code < 0x20: + out += b'\\u%04x' % code + elif 0xD800 <= code <= 0xDFFF: + raise BridgeError("digest", f"string carries an unpaired surrogate U+{code:04X}; " + "serde cannot parse it into a Rust String") + else: + out += ch.encode("utf-8") + out += b'"' + return bytes(out) + + +def _rust_number(text: str) -> str: + """Port of ``store.rs`` ``storage_digest::number`` (coordinator-rs/src/store.rs + :23-51): expand the exponent, trim insignificant fractional zeroes, drop a + negative sign on zero, so PostgreSQL numeric equality (1 == 1.0, -0.0 == 0, + 1e-7 == 0.0000001) collapses to one spelling before hashing. Input is the + serde/Python shortest-round-trip rendering of the number.""" + if "e" in text or "E" in text: + i = text.find("e") + if i == -1: + i = text.find("E") + mantissa, exponent = text[:i], int(text[i + 1:]) + else: + mantissa, exponent = text, 0 + negative = mantissa.startswith("-") + unsigned = mantissa.lstrip("-") + fractional = len(unsigned.split(".", 1)[1]) if "." in unsigned else 0 + digits = unsigned.replace(".", "") + scale = fractional - exponent + if scale <= 0: + digits = digits + "0" * (-scale) + else: + chars = list(digits) + while len(chars) <= scale: + chars.insert(0, "0") + chars.insert(len(chars) - scale, ".") + digits = "".join(chars) + while digits.endswith("0"): + digits = digits[:-1] + if digits.endswith("."): + digits = digits[:-1] + while len(digits) > 1 and digits.startswith("0") and not digits.startswith("0."): + digits = digits[1:] + if negative and digits != "0": + digits = "-" + digits + return digits + + +#: serde_json integer token ranges (no arbitrary_precision): a positive token up +#: to u64::MAX and a negative token down to i64::MIN stay EXACT integers; anything +#: outside becomes f64. +_I64_MIN = -(2 ** 63) +_U64_MAX = 2 ** 64 - 1 + + +def _rust_encode(value: Any, out: bytearray) -> None: + """Port of ``store.rs`` ``storage_digest::encode`` (coordinator-rs/src/store.rs + :52-83). NB: check ``bool`` before ``int`` — a Python bool is an int but serde + renders it in the ``_ =>`` branch as ``true``/``false``, not as a Number.""" + if isinstance(value, bool): + out += b"true" if value else b"false" + elif isinstance(value, int): + # serde_json::Value (no arbitrary_precision) admits an integer token as + # i64 or u64 and keeps it EXACT; a value outside [-2^63, 2^64-1] takes a + # floating representation before number(n.to_string()) (store.rs:56). Match + # that dispatch: exact string in range, else the serde f64 rendering. + if _I64_MIN <= value <= _U64_MAX: + out += _rust_number(str(value)).encode() + else: + try: + f = float(value) + except OverflowError: + raise BridgeError("digest", f"integer {value} exceeds the f64 range serde admits") from None + if not math.isfinite(f): + raise BridgeError("digest", f"integer {value} is not finite as f64") + out += _rust_number(repr(f)).encode() + elif isinstance(value, float): + out += _rust_number(repr(value)).encode() + elif isinstance(value, str): + out += _serde_json_string(value) + elif value is None: + out += b"null" + elif isinstance(value, list): + out += b"[" + for i, el in enumerate(value): + if i: + out += b"," + _rust_encode(el, out) + out += b"]" + elif isinstance(value, dict): + out += b"{" + # serde_json's Map is a BTreeMap (no preserve_order feature): keys sorted. + for i, k in enumerate(sorted(value)): + if i: + out += b"," + out += _serde_json_string(k) + out += b":" + _rust_encode(value[k], out) + out += b"}" + else: + raise BridgeError("digest", f"cannot storage-digest value of type {type(value).__name__}") + + +def _canonical_payload_digest(data: Any) -> str: + """The store's CANONICAL payload digest — a BYTE-FOR-BYTE port of the Rust + ``storage_digest`` (coordinator-rs/src/store.rs:21-89), DISTINCT from the + original-byte hash. It hashes the payload under PostgreSQL's numeric + normalization (expanded exponents, trimmed fractional zeroes, no negative + zero) with serde string/Unicode encoding and BTreeMap-sorted object keys, so + the digest equals the one the Rust store persists in the checkpoint.""" + out = bytearray() + _rust_encode(data, out) + return hashlib.sha256(bytes(out)).hexdigest() + + +def validate_readback(bundle: ReadbackBundle, *, expected_prior_tick: Optional[int], + operation_id: str, publisher_epoch: int, + expected_zid: Optional[int] = None, + expected_math_env: Optional[str] = None, + expected_input_checkpoint: Optional[dict[str, Any]] = None) -> list[str]: + """Grade a Rust producer's own readback of the four math rows (brief §4). + + Requires: publication SCOPE (a non-boolean integer ``zid`` and a nonempty + ``math_env``, bound to ``expected_zid``/``expected_math_env`` when supplied); + all four ``math_tick`` present as NON-BOOLEAN integers and in agreement; + expected prior->next tick (first publication 0 when ``expected_prior_tick is + None``); both companions present with ``data``; epoch/operation binding; + per-table original-byte hashes matching the row's own ``original_sha256`` AND + ``ticks.original_digests``; and TYPE-AWARE JSONB correspondence to those exact + original bytes. + + JSONB evidence (``data``) is REQUIRED per table; ``data::text`` is never + original evidence (a row without ``original_bytes`` fails rather than falling + back to its rendering). ``caching_tick`` is inspected only on ``math_main``; + neither companion carries one. Cross-producer allocation values/owner UUIDs + are out of scope (they need not match); the logical cut mapping that must + match is graded by the campaign sidecar.""" + # Total admission at the public boundary: a non-object bundle is graded, not + # dereferenced. + if not isinstance(bundle, dict): + return [f"readback: bundle must be an object, got {bundle!r}"] + fails: list[str] = [] + + # 0. publication scope + zid = bundle.get("zid") + if not _plain_int(zid): + fails.append(f"bundle zid must be a (non-boolean) integer, got {zid!r}") + elif expected_zid is not None and zid != expected_zid: + fails.append(f"bundle zid {zid!r} != expected {expected_zid!r}") + env = bundle.get("math_env") + if not _nonempty_str(env): + fails.append(f"bundle math_env must be a nonempty string, got {env!r}") + elif expected_math_env is not None and env != expected_math_env: + fails.append(f"bundle math_env {env!r} != expected {expected_math_env!r}") + + ticks = bundle.get("ticks") if isinstance(bundle.get("ticks"), dict) else {} + tick_val = ticks.get("math_tick") + + # 1. next-tick allocation (non-boolean integer) + if not _plain_int(tick_val): + fails.append(f"ticks.math_tick must be a (non-boolean) integer, got {tick_val!r}") + elif expected_prior_tick is None: + if tick_val != 0: + fails.append(f"first publication math_tick must be 0, got {tick_val!r}") + elif tick_val != expected_prior_tick + 1: + fails.append(f"math_tick must be prior+1 ({expected_prior_tick}+1), got {tick_val!r}") + + # 2. four-row tick agreement + companion presence + JSONB present + for name in _PAYLOAD_TABLES: + row = bundle.get(name) + if not isinstance(row, dict) or not row: + fails.append(f"{name}: row absent or not an object") + continue + rt = row.get("math_tick") + if not _plain_int(rt): + fails.append(f"{name}: math_tick must be a (non-boolean) integer, got {rt!r}") + elif rt != tick_val: + fails.append(f"{name}: math_tick {rt!r} != ticks {tick_val!r}") + if not isinstance(row.get("data"), (dict, list)): + fails.append(f"{name}: JSONB data (correspondence evidence) is required") + for name in _COMPANIONS: + row = bundle.get(name) + if isinstance(row, dict) and row.get("caching_tick") is not None: + fails.append(f"{name}: companion must not carry a caching_tick") + main_row = bundle.get("main") + if isinstance(main_row, dict) and "caching_tick" not in main_row: + fails.append("main: caching_tick absent where it must exist") + + # 3. operation / epoch binding (epoch TYPED before comparison: a JSON bool + # must not match an expected integer via Python's True == 1) + if ticks.get("operation_id") != operation_id: + fails.append(f"ticks.operation_id {ticks.get('operation_id')!r} != {operation_id!r}") + epoch = ticks.get("publisher_epoch") + if not _plain_int(epoch): + fails.append(f"ticks.publisher_epoch must be a (non-boolean) integer, got {epoch!r}") + elif epoch != publisher_epoch: + fails.append(f"ticks.publisher_epoch {epoch!r} != {publisher_epoch!r}") + + # 3b. persisted store checkpoint (polis-coordinator/1) custody, matching the + # store's real load_current checks (store.rs). Total admission via the schema + # (required, typed, non-boolean nonnegative cursor slots, votes/moderation + # only), then VALUE bindings the store performs: embedded publisher_epoch == + # the row/expected epoch; original_digests == ticks; and the CANONICAL + # payload_digests (numeric-normalized JSONB, distinct from original-byte + # hashes) == the canonical digest of each table's data. An expected checkpoint + # binds by TYPE-AWARE equality (PG number spelling normalized, bools distinct). + ckpt = ticks.get("input_checkpoint") + admit = _admit(ckpt, _STORE_CHECKPOINT_SPEC, "input_checkpoint") if ckpt is not None else \ + ["input_checkpoint: persisted store checkpoint is required"] + if admit: + fails.extend(admit) + else: + if ckpt["operation_id"] != operation_id: + fails.append(f"input_checkpoint.operation_id {ckpt['operation_id']!r} != {operation_id!r}") + if ckpt["publisher_epoch"] != publisher_epoch: + fails.append(f"input_checkpoint.publisher_epoch {ckpt['publisher_epoch']!r} != {publisher_epoch!r}") + if not _json_type_equal(ckpt["original_digests"], _dict_or_empty(ticks.get("original_digests"))): + fails.append("input_checkpoint.original_digests disagree with ticks.original_digests") + # The encoder REFUSES a value beyond serde's f64 range with BridgeError; + # catch it here so the public API always returns a graded failure list. + canon, digest_ok = {}, True + for name in _PAYLOAD_TABLES: + d = _row_data(bundle, name) + if d is None: + continue + try: + canon[name] = _canonical_payload_digest(d) + except BridgeError as exc: + fails.append(f"{name}: cannot canonical-digest payload — {exc}") + digest_ok = False + if digest_ok and ckpt["payload_digests"] != canon: + fails.append("input_checkpoint.payload_digests are not the canonical digests of the payloads") + if expected_input_checkpoint is not None and not _json_type_equal(ckpt, expected_input_checkpoint): + fails.append("input_checkpoint does not equal the expected checkpoint (type-aware)") + + # 4. original-byte custody + TYPE-AWARE JSONB correspondence, per table + original_digests = _dict_or_empty(ticks.get("original_digests")) + for name in _PAYLOAD_TABLES: + row = bundle.get(name) + if not isinstance(row, dict) or not row: + continue + raw = row.get("original_bytes") + if raw is None: + fails.append(f"{name}: original_bytes absent — data::text is not original evidence") + continue + # Admit the original-byte evidence type BEFORE converting it. A str + # carrying an unpaired surrogate is not UTF-8-encodable (serde cannot hold + # it in a Rust String); grade it rather than raising UnicodeEncodeError. + if isinstance(raw, str): + try: + raw_bytes = raw.encode() + except UnicodeEncodeError: + fails.append(f"{name}: original_bytes is not valid UTF-8 (unpaired " + f"surrogate) — serde cannot hold it in a Rust String") + continue + elif isinstance(raw, (bytes, bytearray)): + raw_bytes = bytes(raw) + else: + fails.append(f"{name}: original_bytes must be a string or byte sequence, " + f"got {type(raw).__name__}") + continue + digest = hashlib.sha256(raw_bytes).hexdigest() + if row.get("original_sha256") != digest: + fails.append(f"{name}: original_sha256 does not match its original_bytes") + if original_digests.get(name) != digest: + fails.append(f"{name}: ticks.original_digests[{name}] does not match original_bytes") + try: + parsed = json.loads(raw_bytes) + except (ValueError, TypeError): + fails.append(f"{name}: original_bytes is not valid JSON") + continue + if "data" in row and not _json_type_equal(parsed, row["data"]): + fails.append(f"{name}: JSONB data does not correspond to the original bytes") + return fails + + +def _check_bidtopid_against_main(main: Any, bid: Any, zid: Any) -> list[str]: + """Grade the companion against the real ``bidToPid`` wrapper. + + Total admission first (``_BIDTOPID_SPEC`` / ``_MAIN_SPEC``): the wrapper shape, + element types (integer pids, no bools/floats), list ranks and group-row shapes + are all decided by the schema, so a malformed value fails with a GRADED message + and the relational checks below — positional ``bidToPid[i] == + base-clusters.members[i]``, unique base ids, ``count == len(members)``, a + participant PARTITION, and group bids that reference existing base ids — can + never raise. An empty generation is valid.""" + admit = _admit(bid, _BIDTOPID_SPEC, "bidtopid") + _admit(main, _MAIN_SPEC, "main") + if admit: + return [f"observer: {f}" for f in admit] + + base = main["base-clusters"] + ids, members, btp = base["id"], base["members"], bid["bidToPid"] + counts = base.get("count") + out: list[str] = [] + if zid is not None and bid["zid"] != zid: + out.append(f"observer: bidtopid.zid {bid['zid']!r} != bundle zid {zid!r}") + if not (len(btp) == len(ids) == len(members)): + return out + [f"observer: bidToPid ({len(btp)}) not aligned to base-clusters " + f"id/members ({len(ids)}/{len(members)})"] + if len(set(ids)) != len(ids): + out.append(f"observer: base-clusters.id has duplicates: {ids}") + if counts is not None: + if len(counts) != len(members): + out.append("observer: base-clusters.count is not one entry per cluster") + else: + for i, (c, mem) in enumerate(zip(counts, members)): + if c != len(mem): + out.append(f"observer: base-clusters.count[{i}]={c!r} != len(members)={len(mem)}") + seen: set = set() + for i, (bucket, mem) in enumerate(zip(btp, members)): + if bucket != mem: + out.append(f"observer: bidToPid[{i}]={bucket} != base-clusters.members[{i}]={mem} " + f"(positional bid membership mismatch)") + if len(set(bucket)) != len(bucket): + out.append(f"observer: base cluster {i} has duplicate participant(s): {bucket}") + overlap = seen & set(bucket) + if overlap: + out.append(f"observer: participant(s) {sorted(overlap)} appear in more than one base cluster") + seen |= set(bucket) + groups = main.get("group-clusters") + if groups is not None: + idset = set(ids) + for g in groups: + for gbid in g["members"]: + if gbid not in idset: + out.append(f"observer: group cluster references unknown base bid {gbid!r}") + return out + + +def observe_bundle_coherence(bundle: ReadbackBundle, + fold_check: Optional[Callable[..., list[str]]] = None) -> list[str]: + """Independent post-cut coherence of a published bundle (brief §4/§5 observer). + + Refuses ABSENT evidence: ticks and all four rows must be present, each with a + non-boolean integer ``math_tick`` in one agreeing generation and JSONB + ``data``; an empty ``{}`` bundle fails rather than trivially passing four + absent-equals-absent ticks. It validates the ACTUAL serialized companion: + ``bidtopid.data`` is the writer's ``{zid, bidToPid, lastVoteTimestamp}`` + wrapper, resolved positionally through ``main.base-clusters.id`` and required + to equal ``base-clusters.members`` bucket for bucket — so an unrelated invented + map and duplicate/overlapping buckets are rejected, and a legitimate empty + generation is accepted. ``fold_check`` (e.g. the recovery oracle's + ``check_published_against_fold``) is the OPTIONAL full latest-cell fold. + Returns failures (empty == coherent). A live mid-publication observer remains + a slice-3 integration obligation; this pure helper does not claim it.""" + if not isinstance(bundle, dict): + return [f"observer: bundle must be an object, got {bundle!r}"] + fails: list[str] = [] + ticks = bundle.get("ticks") + if not isinstance(ticks, dict) or not _plain_int(ticks.get("math_tick")): + fails.append("observer: ticks.math_tick absent or not a non-boolean integer") + tick_val: Optional[int] = None + else: + tick_val = ticks["math_tick"] + for name in _PAYLOAD_TABLES: + row = bundle.get(name) + if not isinstance(row, dict) or not row: + fails.append(f"observer: {name} row absent or not an object") + continue + rt = row.get("math_tick") + if not _plain_int(rt): + fails.append(f"observer: {name} math_tick absent or not a non-boolean integer") + elif tick_val is not None and rt != tick_val: + fails.append(f"observer: {name} at a different generation than ticks") + if not isinstance(row.get("data"), (dict, list)): + fails.append(f"observer: {name} data absent") + main = _row_data(bundle, "main") + bid = _row_data(bundle, "bidtopid") + fails.extend(_check_bidtopid_against_main(main, bid, bundle.get("zid"))) + if fold_check is not None: + try: + fails.extend(fold_check(main) or []) + except Exception as exc: # noqa: BLE001 - a fold failure is a finding, not a crash + fails.append(f"observer: fold raised {exc!r}") + return fails + + +# --------------------------------------------------------------------------- +# Same-invocation stage sibling (p045-stage-binding/1) + stage context. +# --------------------------------------------------------------------------- +#: Full run/session/plan-scoped identity of one stage-context entry. Scoping to +#: the COMPLETE identity (including run_id, not just the S1-reused +#: compute-0/checkpoint-0 local names) is what stops two fresh runs/sessions from +#: colliding, and keeps the campaign plan digest distinct from S1's per-session +#: schedule. +StageContextKey = tuple[str, str, str, str, str] # (plan_sha256, run_id, session_id, compute_id, checkpoint_id) + +#: The nonempty-string identity fields, in key order. +_STAGE_CONTEXT_IDENTITY = ("plan_sha256", "run_id", "session_id", "compute_id", "checkpoint_id") + + +class StageContextEntry(TypedDict, total=False): + plan_sha256: str + run_id: str + session_id: str + compute_id: str + checkpoint_id: str + global_cut_index: int + semantic_input_digest: str # R-ORACLE digest the worker must reproduce + profile: str + + +_STAGE_CONTEXT_KEYS = frozenset(_STAGE_CONTEXT_IDENTITY) | { + "global_cut_index", "semantic_input_digest", "profile"} + + +def load_stage_context(path: str | Path) -> dict[StageContextKey, StageContextEntry]: + """Load a closed ``p045-stage-context/1`` map keyed on the FULL identity + ``(plan_sha256, run_id, session_id, compute_id, checkpoint_id)``. + + Every entry is fully TYPED before its key is built: the container is an + object, the key set is exactly the closed set (no missing/unknown keys), the + five identity fields are NONEMPTY strings (a null or a list-valued identity is + a graded BridgeError, never an ungraded TypeError), ``global_cut_index`` is a + non-boolean nonnegative integer, ``profile`` is a known campaign profile, and + ``semantic_input_digest`` is nonempty. DUPLICATE full identities are rejected + before insertion. A wrong schema fails.""" + obj = json.loads(Path(path).read_bytes()) + if not isinstance(obj, dict) or obj.get("schema") != STAGE_CONTEXT_SCHEMA: + raise BridgeError("stage-context", f"stage context schema must be {STAGE_CONTEXT_SCHEMA!r}") + entries = obj.get("entries") + if not isinstance(entries, list): + raise BridgeError("stage-context", "entries must be a list") + out: dict[StageContextKey, StageContextEntry] = {} + for i, ent in enumerate(entries): + if not isinstance(ent, dict): + raise BridgeError("stage-context", f"entry {i} is not an object") + if set(ent) != _STAGE_CONTEXT_KEYS: + raise BridgeError("stage-context", f"entry {i} key set must be exactly " + f"{sorted(_STAGE_CONTEXT_KEYS)}, got {sorted(ent)}") + for f in _STAGE_CONTEXT_IDENTITY: + if not _nonempty_str(ent[f]): + raise BridgeError("stage-context", f"entry {i} {f} must be a nonempty string") + cut = ent["global_cut_index"] + if not _plain_int(cut) or cut < 0: + raise BridgeError("stage-context", f"entry {i} global_cut_index must be a nonnegative integer") + if ent["profile"] not in PROFILES: + raise BridgeError("stage-context", f"entry {i} profile {ent['profile']!r} not in {PROFILES}") + if not _nonempty_str(ent["semantic_input_digest"]): + raise BridgeError("stage-context", f"entry {i} semantic_input_digest must be nonempty") + key: StageContextKey = tuple(ent[f] for f in _STAGE_CONTEXT_IDENTITY) # type: ignore[assignment] + if key in out: + raise BridgeError("stage-context", f"duplicate stage-context identity {key}") + out[key] = { + "plan_sha256": ent["plan_sha256"], "run_id": ent["run_id"], + "session_id": ent["session_id"], "compute_id": ent["compute_id"], + "checkpoint_id": ent["checkpoint_id"], "global_cut_index": cut, + "semantic_input_digest": ent["semantic_input_digest"], "profile": ent["profile"], + } + return out + + +def check_stage_context(context: dict[StageContextKey, StageContextEntry], *, + plan_sha256: str, session_id: str, compute_id: str, + checkpoint_id: str, derived_semantic_digest: str, + run_id: Optional[str] = None, + expected_profile: Optional[str] = None, + expected_cut_index: Optional[int] = None) -> list[str]: + """The worker derives the semantic digest from the inputs it ACTUALLY applied + and checks it against the plan-bound context, addressed by the FULL identity + INCLUDING run_id — a foreign run with the same local session/compute/checkpoint + names does not match. run binding is MANDATORY: omitting ``run_id`` fails + closed (an unbound consumer cannot certify a run). A foreign identity, a + mismatched digest, or (when supplied) a wrong profile/cut is rejected — the + plan's expected hash is never stamped onto an unverified source.""" + if not _nonempty_str(run_id): + return ["stage context requires a nonempty run_id for run binding"] + key: StageContextKey = (plan_sha256, run_id, session_id, compute_id, checkpoint_id) + entry = context.get(key) + if entry is None: + return [f"foreign stage context: {key} not in the plan"] + fails: list[str] = [] + if entry["semantic_input_digest"] != derived_semantic_digest: + fails.append(f"semantic input digest mismatch: derived {derived_semantic_digest!r} " + f"!= context {entry['semantic_input_digest']!r}") + if expected_profile is not None and entry["profile"] != expected_profile: + fails.append(f"stage-context profile {entry['profile']!r} != expected {expected_profile!r}") + if expected_cut_index is not None and entry["global_cut_index"] != expected_cut_index: + fails.append(f"stage-context cut {entry['global_cut_index']!r} != expected {expected_cut_index!r}") + return fails + + +def stage_binding_manifest(*, run_id: str, session_id: str, compute_id: str, + checkpoint_id: str, operation_id: str, + input_digest: str, schedule_digest: str, + semantic_input_digest: str, stage_file_sha256: str, + output_file_sha256s: dict[str, str], + binary_sha256: str, worker_tree_sha256: str, + profile: str) -> dict[str, Any]: + """The closed ``p045-stage-binding/1`` SIBLING manifest — never appended to + S1's closed four-file checkpoint. Its sidecar identifies producer ``rust`` + and binds it to its binary/worker hashes; the stage engine label stays ``py`` + (the Python science worker), so nobody relabels Python mathematics as Rust. + The publication receipt must match this record.""" + return { + "schema": STAGE_BINDING_SCHEMA, + "producer": "rust", + "stage_engine": "py", + "run_id": run_id, "session_id": session_id, + "compute_id": compute_id, "checkpoint_id": checkpoint_id, + "operation_id": operation_id, + "input_digest": input_digest, + "schedule_digest": schedule_digest, + "semantic_input_digest": semantic_input_digest, + "stage_file_sha256": stage_file_sha256, + "output_file_sha256s": dict(output_file_sha256s), + "binary_sha256": binary_sha256, + "worker_tree_sha256": worker_tree_sha256, + "profile": profile, + } + + +def capture_rust_stage_sibling(conv: Any, blob: dict[str, Any] | None, *, + step_index: int, semantic_input_digest: str, + out_dir: str | Path, tick: Any = None, + run_root: str | Path | None = None) -> Path: + """Write ONE IMMUTABLE stage sibling for a Rust snapshot, using the very + ``conv`` and already-emitted raw ``blob`` from that operation (brief §5). + + Immutability is enforced, not just implied by distinct directories: the file + is created EXCLUSIVELY; if one already exists for this operation/cut, an + identical retry is idempotent but ANY differing content is refused (a second + snapshot for the same cut can never truncate and replace the first). When + ``run_root`` is given the operation directory must be confined within it. + Reuses ``stages.stage_document`` with the ``py`` engine label; never calls + ``write_stage_documents`` (delete-then-write) or ``run_stage_dump`` (a second + replay). Returns the path to the written stage document.""" + from polismath.replay import stages + if not _plain_int(step_index) or step_index < 0: + raise BridgeError("stage-evidence", f"step_index must be a nonnegative integer, got {step_index!r}") + op_dir = Path(out_dir) + if run_root is not None: + rr = Path(run_root).resolve() + try: + op_dir.resolve().relative_to(rr) + except ValueError: + raise BridgeError("stage-evidence", f"operation dir {op_dir} escapes the run root {rr}") + op_dir.mkdir(parents=True, exist_ok=True) + doc = stages.stage_document(conv, step_index=step_index, + digest=semantic_input_digest, tick=tick, blob=blob, + engine=stages.PY_STAGE_ENGINE) + payload = stages.canonical_json(doc) + data = payload.encode() if isinstance(payload, str) else bytes(payload) + dest = op_dir / f"step-{step_index:03d}.stages.json" + try: + with open(dest, "xb") as fh: + fh.write(data) + except FileExistsError: + if dest.read_bytes() != data: + raise BridgeError( + "stage-evidence", + f"refusing to overwrite immutable stage evidence at {dest}: a differing " + f"sibling already exists for this operation") from None + return dest + + +# =========================================================================== +# Slice-1 reachable reporting path (correction 1). +# +# A REAL runner + report so the `rust` producer can be invoked end-to-end the +# way clj/py can, emitting the required per-entry rows into a +# polis-certification-run/2 report — WITHOUT the blocked forced-compute or warm +# path. Unsupported computations stay non-passing: rust records +# UNSUPPORTED_PROFILE, the references record INCONCLUSIVE (their real recording +# is the existing certify.ensure_*_recording path, not re-run here), and every +# pair is non-passing. A bridge run ALWAYS writes a terminal manifest. +# +# STILL OUTSTANDING for full slice 1 (named, not faked): the Clojure three-table +# companion sink (real cm/prep-main / prep-bidToPid / prep-ptpt-stats capture +# through the pg-json/upload-math-* path) is a real-engine change to +# math/dev/replay.clj and cannot run without the JVM; it is not implemented here. +# =========================================================================== + +BRIDGE_RUN_MANIFEST = "bridge_run_manifest-{run_id}.json" + + +def _entry_producer_rows(expected: Optional["cert.ExpectedEntry"], entry: Any, + profile: str, plan: dict[str, Any], out_root: Path, + drivers: tuple[str, ...]) -> list[dict[str, Any]]: + """Producer rows for one entry, for EXACTLY the selected ``drivers`` — the + report must not iterate all three when a subset was requested. Works whether + or not the entry prepared.""" + if expected is not None: + inv = {r["engine"]: r for r in three_producer_inventory(expected, profile, drivers)} + else: + role = getattr(entry, "role", None) or f"{getattr(entry, 'dataset', '?')}:{getattr(entry, 'schedule_id', '?')}" + inv = {eng: {"dataset": getattr(entry, "dataset", None), + "schedule_id": getattr(entry, "schedule_id", None), "role": role, + "engine": eng, "loader": REGISTRY[eng].loader, "profile": profile, + "coverage": None, "stream_end": None, "checkpoints": [], + "advertised": REGISTRY[eng].supports(profile), + "status": (ProducerStatus.INCONCLUSIVE if REGISTRY[eng].supports(profile) + else ProducerStatus.UNSUPPORTED_PROFILE).value} + for eng in drivers} + rows: list[dict[str, Any]] = [] + for eng in drivers: + row = dict(inv[eng]) + receipt = REGISTRY[eng].record(plan, expected if expected is not None else entry, + out_root / eng) + row["receipt"] = receipt.to_dict() + row["status"] = receipt.status.value + rows.append(row) + return rows + + +def _pair_rows(entry_rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Pair rows for every UNORDERED pair among the producers actually present in + ``entry_rows`` (so a subset selection yields only its own pairs). A pair is + non-passing unless BOTH sides produced a PASS producer row (none can today).""" + by_eng = {r["engine"]: r for r in entry_rows} + present = [e for e in DRIVER_IDS if e in by_eng] + pairs = [] + for i, a in enumerate(present): + for b in present[i + 1:]: + sa, sb = by_eng[a]["status"], by_eng[b]["status"] + status = (ProducerStatus.PASS.value if sa == sb == ProducerStatus.PASS.value + else ProducerStatus.UNSUPPORTED_PROFILE.value + if ProducerStatus.UNSUPPORTED_PROFILE.value in (sa, sb) + else ProducerStatus.INCONCLUSIVE.value) + pairs.append({"a": a, "b": b, "a_status": sa, "b_status": sb, "status": status}) + return pairs + + +def run_bridge_battery(entries: list[Any], *, root: str | Path, profile: str, + drivers: tuple[str, ...] = DRIVER_IDS, + policy_path: Optional[str | Path] = None, + battery_path: Optional[str | Path] = None) -> dict[str, Any]: + """Run the three-producer bridge over ``entries`` and write a terminal + polis-certification-run/2 report to ``/certify_report_bridge.json``. + + No engine is launched and no database is touched; every producer row is + non-passing (rust UNSUPPORTED_PROFILE; references INCONCLUSIVE). Returns the + report dict. Raises BridgeError only on CONFIGURATION failure (bad drivers, + unknown profile, bad policy) — a per-entry preparation failure becomes a + non-passing row, not a crash, and the manifest is still written.""" + import uuid as _uuid + from datetime import datetime, timezone + drivers = parse_drivers(",".join(drivers)) if not isinstance(drivers, str) else parse_drivers(drivers) + if profile not in PROFILES: + raise BridgeError("profile", f"unknown profile {profile!r} (allowed: {PROFILES})") + policy = load_policy(policy_path) if policy_path is not None else None + root = Path(root) + root.mkdir(parents=True, exist_ok=True) + plan = {"profile": profile, "drivers": list(drivers)} + run_id = str(_uuid.uuid4()) + + producers: list[dict[str, Any]] = [] + pairs: list[dict[str, Any]] = [] + inventory: list[dict[str, Any]] = [] + for entry in entries: + try: + expected: Optional[cert.ExpectedEntry] = cert.prepare_entry(entry) + except Exception as exc: # noqa: BLE001 - a bad entry is a non-passing row + expected = None + plan_entry_err = str(exc) + else: + plan_entry_err = None + rows = _entry_producer_rows(expected, entry, profile, plan, root / run_id, drivers) + if plan_entry_err is not None: + for r in rows: + if r["engine"] != "rust": + r["status"] = ProducerStatus.INCONCLUSIVE.value + r["reason"] = f"prepare failed: {plan_entry_err}" + producers.extend(rows) + inventory.extend({k: r[k] for k in ("dataset", "schedule_id", "engine", "loader", + "profile", "advertised", "status")} for r in rows) + pairs.extend({"dataset": getattr(entry, "dataset", None), + "schedule_id": getattr(entry, "schedule_id", None), **p} + for p in _pair_rows(rows)) + + all_pass = bool(producers) and all(r["status"] == ProducerStatus.PASS.value for r in producers) + verdict = ProducerStatus.PASS.value if all_pass else ProducerStatus.INCONCLUSIVE.value + report = { + "schema": CERTIFICATION_RUN_SCHEMA_V2, + "run_id": run_id, + "finished_at": datetime.now(timezone.utc).isoformat(), + "profile": profile, + "drivers": list(drivers), + "battery_path": str(battery_path) if battery_path is not None else None, + "policy": policy.to_dict() if policy is not None else None, + "inventory": inventory, + "producers": producers, + "pairs": pairs, + "verdict": verdict, + "terminal": True, + "notes": ["rust producer UNSUPPORTED_PROFILE until the slice-3 " + "p045-replay-plan/1 forced-compute path exists", + "reference producers INCONCLUSIVE: their real recording is the " + "existing certify.ensure_*_recording path, not re-run here", + "Clojure three-table companion sink is a real-engine change not " + "implemented in this reachable-path slice"], + } + manifest_path = root / "certify_report_bridge.json" + manifest_path.write_text(json.dumps(report, indent=2, sort_keys=True, default=str)) + (root / BRIDGE_RUN_MANIFEST.format(run_id=run_id)).write_text( + json.dumps(report, indent=2, sort_keys=True, default=str)) + report["report_path"] = str(manifest_path) + return report + + +def render_bridge_lines(report: dict[str, Any]) -> list[str]: + """Human summary of a bridge report for the CLI.""" + lines = [f"bridge {report['schema']} verdict={report['verdict']} " + f"profile={report['profile']} drivers={','.join(report['drivers'])}"] + for r in report["producers"]: + lines.append(f" {r['dataset']}:{r['schedule_id']} {r['engine']:>4} -> {r['status']}") + return lines + + +def bridge_exit_code(report: dict[str, Any]) -> int: + """0 PASS, 1 FAIL, 2 INCONCLUSIVE (incl. UNSUPPORTED_PROFILE).""" + v = report.get("verdict") + if v == ProducerStatus.PASS.value: + return 0 + if v == ProducerStatus.FAIL.value: + return 1 + return 2 diff --git a/delphi/scripts/certify.py b/delphi/scripts/certify.py index aa6c3241a9..ca525580db 100644 --- a/delphi/scripts/certify.py +++ b/delphi/scripts/certify.py @@ -57,8 +57,41 @@ def cli() -> None: @click.option("--workers", type=int, default=6, show_default=True, help="Parallel battery entries (drivers + compare); the ledger " "fold stays serial, so results match --workers 1 exactly.") -def run(battery_path, only, refresh_clj, refresh_py, strict, root, workers): - """Certify every entry in the battery (or a filtered subset).""" +@click.option("--drivers", default="clj,py", show_default=True, + help="Comma-separated producer ids (clj,py[,rust]). The default " + "clj,py is the unchanged legacy two-driver run; selecting rust " + "(or any non-default set) routes to the P-045 bridge, which " + "emits a polis-certification-run/2 report with per-entry " + "clj/py/rust rows. rust requires both references.") +@click.option("--profile", "profile", default=None, + help="Bridge campaign profile (battery-chain/1 or " + "snapshot-rebuild/1); mandatory when rust is selected.") +def run(battery_path, only, refresh_clj, refresh_py, strict, root, workers, drivers, profile): + """Certify every entry in the battery (or a filtered subset). + + ``--drivers clj,py`` with no ``--profile`` is the legacy two-driver path, + byte-for-byte unchanged. Any other driver set routes to the additive P-045 + bridge, whose rust rows are UNSUPPORTED_PROFILE until the slice-3 + forced-compute path exists (exit 2 / INCONCLUSIVE, never a fabricated PASS). + """ + if drivers != "clj,py" or profile is not None: + from polismath.replay import coordinator_driver as cdrv + try: + selected = cdrv.parse_drivers(drivers) + if profile is None: + raise cdrv.BridgeError("profile", "--profile is required for a bridge run") + entries = cert.load_battery(battery_path) + if only is not None: + entries = cert._filter_only(entries, only) + report = cdrv.run_bridge_battery( + entries, root=root or cert.st.replays_root(), profile=profile, + drivers=selected, battery_path=battery_path) + except (cdrv.BridgeError, ValueError, KeyError, TypeError, OSError) as exc: + click.echo(f"certify: bridge FAIL [configuration] {exc}") + sys.exit(2) + for line in cdrv.render_bridge_lines(report): + click.echo(line) + sys.exit(cdrv.bridge_exit_code(report)) try: entries = cert.load_battery(battery_path) report = cert.run_battery(entries, root=root, refresh_clj=refresh_clj, diff --git a/delphi/tests/coordinator/__init__.py b/delphi/tests/coordinator/__init__.py new file mode 100644 index 0000000000..5b9065391f --- /dev/null +++ b/delphi/tests/coordinator/__init__.py @@ -0,0 +1,10 @@ +"""Package marker, and it is load-bearing. + +Without it these files are imported by pytest as top-level modules, so this +directory's `conftest.py` claims the module name `conftest` and shadows +`delphi/tests/conftest.py` for every sibling test module that does +`from conftest import ...`. In the Delphi CI layout (tests copied to +`/app/tests`) that broke four unrelated modules at collection. As a package the +modules here are `coordinator.*`, the parent conftest keeps the plain name, and +the imports below are explicit about which conftest they mean. +""" diff --git a/delphi/tests/coordinator/audit_stages.py b/delphi/tests/coordinator/audit_stages.py new file mode 100644 index 0000000000..f536525961 --- /dev/null +++ b/delphi/tests/coordinator/audit_stages.py @@ -0,0 +1,123 @@ +"""Emit an honest CO07 inventory; missing contract stages block certification.""" +import json +import hashlib +from pathlib import Path +import subprocess +import sys + +root=Path(__file__).resolve().parents[3] +artifacts=root/'coordinator-rs/artifacts' +compiled=json.loads(subprocess.check_output([str(root/'coordinator-rs/target/fault/debug/polis-coordinator'),'stages'],text=True)) +required=['after_source_selection','after_input_checkpoint','before_worker_apply','after_worker_compute', + 'after_lease','before_ticks','after_ticks','before_bidtopid','after_bidtopid','before_ptptstats','after_ptptstats', + 'before_main','after_main','before_commit','after_commit','after_ack','before_restore','after_restore', + 'before_cursor','after_cursor','before_sweep','after_sweep','bundle_pinned','before_companion_join', + 'cache_eviction_contends_with_same_zid_update'] +rows=[] +for stage in required: + file=artifacts/f'stage-{stage}.json' + if stage in compiled and file.exists(): + evidence=json.loads(file.read_text()) + assert evidence['ack']['stage']==stage + assert evidence['ack']['state']=='reached-and-blocked' + assert evidence.get('postcondition') + rows.append(dict(stage=stage,status='reached-and-blocked',evidence=evidence)) + else: + rows.append(dict(stage=stage,status='required-but-unreached',reason='no evidence artifact for a compiled, contract-required stage')) +assert not set(compiled)-set(required), 'unreviewed compiled marker' +# The stage inventory and the full contract gate are separate verdicts: reaching +# every fault stage does not certify CO08/D4, which need the actual Node route. +open_conditions=[ + 'CO08/D4: the real Node reader (getPca + getBidIndexToPidMapping + getPidsForGid) serves' + ' identical Rust/Python bytes at generation one, and the real pca2 route serves' + ' generation zero, but CO04\'s loadBundle/Bundle cache-unit rewrite does not exist in' + ' the server and is untested here', + 'CO08/D4: application boot, auth, report.ts/doFamousQuery and the private 2,884-case' + ' served corpus are not executed; the empty presentation is compared synthetically', + 'CO08/D4 reader defect (server owner): getPca(zid, undefined) misses a cold committed' + ' generation zero and finds it once the route warms the cache; the route itself,' + ' which passes math_tick=-1, serves it with 200 and ETag "0"', + 'C7 / polis-empty-served/1: the Python comparison writer publishes no row for a zero-vote' + ' conversation, the server synthesizes the approved-comment listing with a' + ' request-clock lastVoteTimestamp, and a published empty generation lists no tids;' + ' the transition must preserve comment ownership; the contract owner must still' + ' admit a frozen clock or path-specific clock normalization before this gate closes', + 'CO01: the incremental probe is a hint bounded by P026_RECONCILE_SECONDS, measured' + ' from before the source read; the interval makes a conversation eligible for' + ' reconciliation and is not a deadline, and no pass/service budget has been measured', + 'CO02/CO06: persisted payloads become eligible for revalidation once the ceiling' + ' elapses, not on every pass; resident-cache reconciliation is single-threaded and' + ' no multi-worker or warm-worker cache campaign exists', + 'P-031: this crate implements none of A01 PollHealthy, A02 PublishLagSeconds or' + ' A03 ObserverHealthy, has no deployed publisher and no delivery proof; the optional' + ' synchronous sink can still block when enabled', + 'O8 remains OPEN and is at most PARTIAL: polis-candidate-input/1 is claimed locally as a' + ' candidate profile, not a G01-G16 certificate. S1 records the distinct candidate schema id,' + ' strict adapter admission, original-byte digests with decoded correspondence, and the' + ' worker/engine pins; still outstanding are all applicable G01-G16 cases with their required' + ' negative controls, an immutable manifest, the distinction between fresh rebuild, exact' + ' resume and warm incremental output schedules, and a non-vacuous P-023 compensated pair with' + ' raw C9 validation'] +# S1 records part of the local candidate-identity/storage obligation. It does not +# close O8: the plan's O8 row also requires the G01-G16 case set, an immutable +# manifest, the three output schedules and the P-023 pair, none of which S1 +# delivers. So the standing G01-G16 disclaimer above is never removed, and the +# best state this script will ever write for O8 is PARTIAL. +# +# Reproducibility: the closure may pin only committed repository inputs — source, +# migration, schemas, tests. Build outputs under coordinator-rs/target/ and run +# logs under coordinator-rs/artifacts/ are gitignored, so hash-gating on them +# would make the committed inventory verifiable on exactly one machine and +# self-invalidating everywhere else. Those hashes are still recorded, but as an +# informational `run_pins` block that this gate deliberately does not check. +IGNORED_PREFIXES = ('coordinator-rs/target/', 'coordinator-rs/artifacts/') +condition_states = {f'O{i}': 'OPEN' for i in range(1, 9)} +partial_conditions = [] +closure_path = root/'coordinator-rs/evidence/s1-closure.json' +if closure_path.exists(): + closure = json.loads(closure_path.read_text()) + try: + assert closure['id'] == 'O8' and closure['scope'] == 'P-026 step-4 S1 candidate identity and original-byte custody' + assert closure['state'] == 'PARTIAL' and closure['g01_g16_certified'] is False + assert closure['remaining_obligations'], 'a PARTIAL record must name what is still open' + required_pins = { + 'coordinator-rs/src/engine.rs', 'coordinator-rs/src/store.rs', 'coordinator-rs/src/main.rs', + 'coordinator-rs/migration.sql', 'coordinator-rs/Cargo.lock', 'coordinator-rs/rust-toolchain.toml', + 'coordinator-rs/schemas/candidate-admission.schema.json', + 'delphi/polismath/engine_adapter.py', 'delphi/tests/coordinator/test_s1_identity.py', + 'delphi/tests/coordinator/test_adapter.py', 'delphi/tests/coordinator/audit_stages.py', + 'coordinator-rs/tests/s1_admission.rs', 'coordinator-rs/tools/record_s1.py', + } + assert required_pins <= closure['sha256'].keys(), 'missing S1 source pins' + for name, expected in sorted(closure['sha256'].items()): + assert not name.startswith(IGNORED_PREFIXES), f'gitignored build output pinned as a source: {name}' + assert hashlib.sha256((root/name).read_bytes()).hexdigest() == expected, name + partial_conditions.append(dict(id='O8', state='PARTIAL', + standing_condition=open_conditions[-1], evidence=str(closure_path.relative_to(root)), + scope=closure['scope'], recorded=closure['recorded'], + remaining_obligations=closure['remaining_obligations'], + g01_g16_certified=False, run_pins_verified_here=False)) + condition_states['O8'] = 'PARTIAL (S1 identity/custody recorded; G01-G16 open)' + except (AssertionError, KeyError, OSError, ValueError) as error: + print(f'S1 partial record refused: {error}', file=sys.stderr) + +metrics=json.loads(subprocess.check_output([str(root/'coordinator-rs/target/fault/debug/polis-coordinator'),'metrics'],text=True)) +assert metrics['namespace']=='Polis/Math' and metrics['dimensions']==['Environment','MathEnv'] +# Rev7 observability admission: no row may claim a P-031 alarm this crate does not implement. +assert metrics['p031_status']['coverage_claimed']==[] +assert all(m['p031_alarm']=='' for m in metrics['metrics']) +unreached=[r['stage'] for r in rows if r['status']!='reached-and-blocked'] +result=dict(protocol='polis-fault-control/1',compiled_stages=len(compiled),required_stages=len(required), + metrics_namespace=metrics['namespace'],declared_metrics=len(metrics['metrics']), + reached=sum(r['status']=='reached-and-blocked' for r in rows),stages=rows,unreached=unreached, + stage_inventory_gate='FAIL' if unreached else 'PASS', + open_conditions=open_conditions, + condition_states=condition_states,closed_conditions=[],partial_conditions=partial_conditions, + full_contract_gate='FAIL', + profile='Rust coordinator + Python worker + CLI Bundle reader, the real Node ' + 'getPca/getBidIndexToPidMapping reader in-process, and the real pca2 route ' + 'over loopback HTTP; no loadBundle, no application boot') +path=root/'coordinator-rs/evidence/stage-inventory.json' +path.write_text(json.dumps(result,indent=2)+'\n') +print(json.dumps({k:v for k,v in result.items() if k!='stages'})) +sys.exit(1 if result['full_contract_gate']=='FAIL' else 0) diff --git a/delphi/tests/coordinator/conftest.py b/delphi/tests/coordinator/conftest.py new file mode 100644 index 0000000000..9da1791ed4 --- /dev/null +++ b/delphi/tests/coordinator/conftest.py @@ -0,0 +1,316 @@ +"""Real migrated PG, process launcher and pinned independent C assertion assets. + +Collection safety. The Delphi CI job copies `delphi/tests` into the delphi image +at `/app/tests`, where neither `coordinator-rs/` nor a repository checkout +exists. Everything here needs both — the crate's built binary, the crate's +migration, and a read-only `git show` of the pinned reference fold/mapping +assets — so this conftest locates the checkout by walking up for +`coordinator-rs/Cargo.toml` (override: `POLIS_CHECKOUT_DIR`) and, when it cannot, +ignores this whole directory with a clear reason instead of raising while +collecting. An explicit `POLIS_CHECKOUT_DIR` that does not resolve is a hard +error rather than a silent skip. When the checkout IS present nothing here is +weakened: every fixture and assertion behaves exactly as before. +""" +import ast +import importlib.util +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import sys +import time +import uuid + +import psycopg2 +import pytest +import sqlalchemy as sa + +_HERE = Path(__file__).resolve() +_MARKER = "coordinator-rs/Cargo.toml" + + +def _locate_checkout(): + """Return (root, skip_reason, fatal_reason). Never raises, never exits.""" + override = os.environ.get("POLIS_CHECKOUT_DIR") + if override: + root = Path(override).expanduser() + if (root / _MARKER).is_file(): + return root.resolve(), None, None + return None, None, ( + f"POLIS_CHECKOUT_DIR={override!r} does not contain {_MARKER}; unset it " + "or point it at a polis checkout" + ) + for candidate in _HERE.parents: + if (candidate / _MARKER).is_file(): + return candidate, None, None + return None, ( + f"no polis checkout containing {_MARKER} above {_HERE.parent}: these tests need " + "the coordinator-rs crate, its migration and the pinned git reference assets " + "(set POLIS_CHECKOUT_DIR to run them from a copied test tree)" + ), None + + +ROOT, _SKIP, _FATAL = _locate_checkout() +if ROOT is not None and _SKIP is None: + # Either a Rust toolchain or an already-built fault binary. A missing + # binary with cargo present is still a loud failure, not a skip. + if (shutil.which("cargo") is None + and not (ROOT / "coordinator-rs/target/fault/debug/polis-coordinator").exists()): + _SKIP = ("neither cargo nor a built coordinator-rs fault binary is available, so the " + "process these tests launch cannot exist here") + elif shutil.which("git") is None or not (ROOT / ".git").exists(): + _SKIP = (f"git or {ROOT}/.git is unavailable, so the pinned independent fold and " + "R09 mapping assets cannot be read") +_UNAVAILABLE = _SKIP or _FATAL +if _UNAVAILABLE: + # Nothing in this directory is collected, so no test module imports the + # names below and no assertion is silently weakened. + collect_ignore_glob = ["*"] + + def pytest_report_collectionfinish(config): + return f"coordinator tests not collected: {_UNAVAILABLE}" + + +def pytest_configure(config): + # Neither import nor collection time: an unusable explicit override fails + # the run cleanly instead of disappearing into a skip. + if _FATAL: + raise pytest.UsageError(_FATAL) + + +ROOT = ROOT if ROOT is not None else _HERE.parents[2] +BINARY = ROOT / "coordinator-rs/target/fault/debug/polis-coordinator" +REFERENCE = "aaaf7ca5c93f9a758b28e7a361c3b9544e24305a" +WRITER_REF = "b3262008f" +ARTIFACTS = ROOT / "coordinator-rs/artifacts" +if not _UNAVAILABLE: + ARTIFACTS.mkdir(exist_ok=True) + + +def asset(name): + return subprocess.check_output(["git", "show", f"{REFERENCE}:delphi/tests/poller/recovery/{name}"], cwd=ROOT, text=True) + + +def oracle_module(): + import types + mod = types.ModuleType("p026_independent_fold") + sys.modules[mod.__name__] = mod + exec(compile(asset("fold.py"), "pinned-fold.py", "exec"), mod.__dict__) + return mod + + +# Guarded only so that an uncollectable tree (see the module docstring) cannot +# raise from `git show` while pytest is still importing conftests. +FOLD = MAPPING = None +if not _UNAVAILABLE: + FOLD = oracle_module() + _tree = ast.parse(asset("test_r09_partial_tables_readers.py")) + _mapping = next(n for n in _tree.body if isinstance(n, ast.FunctionDef) and n.name == "_mapping_problems") + _scope = {} + exec(compile(ast.Module(body=[_mapping], type_ignores=[]), "pinned-r09.py", "exec"), _scope) + MAPPING = _scope["_mapping_problems"] + + +def connect(url): + c = psycopg2.connect(url) + c.autocommit = True + return c + + +@pytest.fixture(scope="session") +def template(): + base = os.environ["POLIS_TEST_POSTGRES_URL"] + name = "p026_template_" + uuid.uuid4().hex[:10] + admin = connect(base) + with admin.cursor() as cur: + cur.execute(f'CREATE DATABASE "{name}"') + admin.close() + url = base.rsplit("/", 1)[0] + "/" + name + c = connect(url) + try: + with c.cursor() as cur: + for path in sorted((ROOT / "server/postgres/migrations").glob("*.sql")): + cur.execute(path.read_text()) + # Exercise the production drift named in CO04: ticks lacks this column. + cur.execute("ALTER TABLE math_ticks DROP COLUMN caching_tick") + cur.execute((ROOT / "coordinator-rs/migration.sql").read_text()) + cur.execute("CREATE TABLE p026_test_marker(namespace text primary key)") + c.close() + yield base, name + finally: + c.close() + admin = connect(base) + with admin.cursor() as cur: + cur.execute(f'DROP DATABASE "{name}" WITH (FORCE)') + admin.close() + + +@pytest.fixture +def db(template): + base, template_name = template + name = "p026_case_" + uuid.uuid4().hex[:10] + admin = connect(base) + with admin.cursor() as cur: + cur.execute(f'CREATE DATABASE "{name}" TEMPLATE "{template_name}"') + url = base.rsplit("/", 1)[0] + "/" + name + c = connect(url) + with c.cursor() as cur: + for env in ("rustproto", "python", "positive", "negative", "reader", "recovery"): + cur.execute("INSERT INTO p026_test_marker VALUES(%s)", (env,)) + c.close() + yield url + with admin.cursor() as cur: + cur.execute(f'DROP DATABASE "{name}" WITH (FORCE)') + admin.close() + + +def seed(url, zid=1, n_ptpts=6, n_cmts=4, votes=True): + c = connect(url) + with c.cursor() as cur: + cur.execute("SET session_replication_role=replica") + cur.execute("INSERT INTO conversations(zid,topic) VALUES(%s,'Synthetic P026')", (zid,)) + for pid in range(n_ptpts): + cur.execute("INSERT INTO participants(zid,pid,uid,mod) VALUES(%s,%s,%s,0)", (zid,pid,100000*zid+pid)) + for tid in range(n_cmts): + cur.execute("INSERT INTO comments(zid,tid,pid,uid,txt,mod,is_meta,created,modified) VALUES(%s,%s,0,100000,%s,0,false,1000,1000)", (zid,tid,f'Synthetic {tid}')) + if votes: + for pid in range(n_ptpts): + for tid in range(n_cmts): + value = [-1,1,0][(pid+tid)%3] + cur.execute("INSERT INTO votes(zid,pid,tid,vote,created) VALUES(%s,%s,%s,%s,%s)", (zid,pid,tid,value,1000+pid*n_cmts+tid)) + c.close() + + +def rows(url, zid=1, env="rustproto"): + c = connect(url) + with c.cursor() as cur: + cur.execute("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + result = {} + for table in ("math_main","math_bidtopid","math_ptptstats","math_ticks"): + cur.execute(f"SELECT * FROM {table} WHERE zid=%s AND math_env=%s", (zid,env)) + r = cur.fetchone() + result[table] = dict(zip([d[0] for d in cur.description], r)) if r else None + cur.execute("COMMIT") + c.close() + return result + + +def assert_coherent(url, zid=1, env="rustproto", fold=True): + tables = rows(url,zid,env) + assert all(tables.values()), tables + assert len({r["math_tick"] for r in tables.values()}) == 1, tables + main,bid,stats = [tables[k]["data"] for k in ("math_main","math_bidtopid","math_ptptstats")] + assert MAPPING(main,bid,stats) == [] + if fold: + c = connect(url) + with c.cursor() as cur: + cur.execute("SELECT pid,tid,vote,created FROM votes WHERE zid=%s ORDER BY created,tid,pid,vote", (zid,)) + events = [dict(zip(("pid","tid","vote","created"), r)) for r in cur] + c.close() + expected = FOLD.fold_votes(events) + assert FOLD.check_published_against_fold(main, expected) == [] + assert tables["math_ticks"]["input_checkpoint"]["event_count"] == len(events) + return tables + + +class Child: + def __init__(self, url, mode="once", env="rustproto", stage=None, directory=None, extra=None, args=()): + self.directory = Path(directory) if directory else None + self.out = self.err = None + process_env = dict(os.environ, DATABASE_URL=url, MATH_ENV=env, P026_PYTHON=sys.executable, + PYTHONPATH=str(ROOT/"delphi"), OMP_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1", MKL_NUM_THREADS="1", + PYTHONDONTWRITEBYTECODE="1", P026_PAGE_SIZE="2", P026_WINDOW="1", P026_LEASE_SECONDS="120") + process_env.pop("P026_FAULT_DIR",None) + if extra: + process_env.update(extra) + if stage: + self.directory.mkdir(parents=True,exist_ok=True) + (self.directory/"arm.json").write_text(json.dumps({"protocol":"polis-fault-control/1", + "stage":stage,"run_id":"synthetic-test","operation_id":uuid.uuid4().hex})) + process_env["P026_FAULT_DIR"] = str(self.directory) + self.proc = subprocess.Popen([str(BINARY),mode,*map(str,args)], env=process_env, + stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,cwd=ROOT/"delphi") + + def ack(self): + deadline = time.monotonic()+150 + path = self.directory/"ack.json" + while time.monotonic()clock_timestamp() FROM coordinator_leases WHERE math_env=%s AND zid=%s", (env, zid)) + row = cur.fetchone() + c.close() + return dict(zip(("owner_id", "owner_epoch", "unexpired"), row)) if row else None + + +def wait(predicate, timeout=90, alive=None, why="condition"): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + value = predicate() + if value: + return value + if alive is not None: + assert alive.proc.poll() is None, (why, alive.proc.poll()) + time.sleep(.05) + pytest.fail(f"{why} not reached in {timeout}s") + + +def repair_after_unclean_death(launch, db, predicate, lease_seconds="2", timeout=90, why="repair"): + """Restart after a SIGKILL without ever expiring the dead owner's lease. + + The restarted process must defer the conversation while the dead owner's + lease is genuinely live and repair once it elapses in database time. + """ + child = launch(db, "run", extra={"P026_LEASE_SECONDS": lease_seconds, "P026_POLL_MS": "50"}) + wait(predicate, timeout=timeout, alive=child, why=why) + return child.kill() + + +def expire(url): + c = connect(url) + with c.cursor() as cur: + cur.execute("UPDATE coordinator_leases SET expires_at=clock_timestamp()-interval '1 second'") + c.close() diff --git a/delphi/tests/coordinator/reference_child.py b/delphi/tests/coordinator/reference_child.py new file mode 100644 index 0000000000..5b2e0baf00 --- /dev/null +++ b/delphi/tests/coordinator/reference_child.py @@ -0,0 +1,48 @@ +"""Pinned #2704 poller, one explicitly scheduled cold/rebuild checkpoint. + +Reads the same local DB; the parent controls visibility cuts. The static comment +snapshot is read by load-or-init; there is no second moderation-triggered compute. +""" +import importlib +import logging +import os +from pathlib import Path +import subprocess +import sys +import tempfile + +ROOT=Path(__file__).resolve().parents[3] +REF="b3262008f" +logging.disable(logging.CRITICAL) +with tempfile.TemporaryDirectory(prefix="p026-reference-") as tmp: + tmp=Path(tmp) + import polismath.database + import polismath.poller + for folder,names in [("database",["postgres.py"]),("poller",["service.py","worker_pool.py","math_writer.py"])]: + dest=tmp/folder;dest.mkdir() + for name in names: + (dest/name).write_bytes(subprocess.check_output(["git","show",f"{REF}:delphi/polismath/{folder}/{name}"],cwd=ROOT)) + getattr(polismath,folder).__path__.insert(0,str(dest)) + for name in ("polismath.database.postgres", "polismath.poller.worker_pool", "polismath.poller.math_writer", "polismath.poller.service"): + sys.modules.pop(name,None) + from polismath.database.postgres import PostgresClient,PostgresConfig + from polismath.poller.service import MathPollerService,PollerConfig + assert str(tmp) in sys.modules["polismath.poller.math_writer"].__file__ + assert str(tmp) in sys.modules["polismath.database.postgres"].__file__ + pg=PostgresClient(PostgresConfig(url=os.environ["DATABASE_URL"],math_env="python",ssl_mode="disable")) + svc=MathPollerService(pg,PollerConfig(math_env="python",worker_pool_size=1,retry_cap=0,allowlist=[1])) + svc._ensure_runtime() + svc._vote_wm=0 + svc._mod_wm=2**63-1 + svc.poll_once() + assert svc._pool.join(timeout=120) + assert not svc._pool.parked_zids(), "reference failed/parked" + if "--warm-boundary" in sys.argv: + import sqlalchemy as sa + t=svc._vote_wm + with pg.engine.begin() as conn: + conn.execute(sa.text("INSERT INTO votes(zid,pid,tid,vote,created) VALUES(1,0,0,1,:t),(1,2,2,-1,:later)"),{"t":t,"later":t+1000}) + svc.poll_once() + assert not svc._pool.parked_zids() + svc.stop();pg.shutdown() +print("DONE") diff --git a/delphi/tests/coordinator/test_adapter.py b/delphi/tests/coordinator/test_adapter.py new file mode 100644 index 0000000000..240ed4c531 --- /dev/null +++ b/delphi/tests/coordinator/test_adapter.py @@ -0,0 +1,153 @@ +import json +from pathlib import Path +import pytest +from polismath.engine_adapter import Adapter, ProtocolError, descriptor, strict_json, CANDIDATE_SCHEMA, ENGINE_VERSION + + +def setup(tmp_path, votes=None, ops=None, sign=-1): + src=tmp_path/"in"; dst=tmp_path/"out"; src.mkdir() + votes = votes or [] + rows=[dict(slot=i+1,source_ordinal=i,stream_ordinal=i,created_ms=1000+i,pid=2,tid=3,raw_vote=v,weight_x_32767=None) for i,v in enumerate(votes)] + (src/"votes.jsonl").write_text(''.join(json.dumps(v)+'\n' for v in rows)) + (src/"mods.jsonl").write_text('') + manifest=dict(schema=CANDIDATE_SCHEMA,fixture_id="synthetic-empty",storage_agree_value=sign,ordering="frozen-extract-order",votes=descriptor(src,"votes.jsonl"),moderation=descriptor(src,"mods.jsonl"),parent=None) + (src/"manifest.json").write_text(json.dumps(manifest)) + (src/"schedule.json").write_text(json.dumps({"schema":"polis-schedule/1","operations":ops or []})) + p=dict(input_manifest=descriptor(src,"manifest.json"),resolved_schedule=descriptor(src,"schedule.json"),required_capabilities=["rebuild-prefix/1"],config=dict(profile="candidate-profile",seed=42,pca_mode="powerit",empty_contract=True,init_vector="ones")) + p["admission"] = dict(candidate_schema=CANDIDATE_SCHEMA, engine_version=ENGINE_VERSION, + input_digest=p["input_manifest"]["sha256"], + schedule_digest=p["resolved_schedule"]["sha256"], operation_id="synthetic-operation") + return Adapter(src,dst),p + + +def req(op,p,rid=1): + return dict(protocol="polis-engine/1",run_id="synthetic-run-001",session_id="synthetic-session-001",request_id=rid,op=op,payload=p) + + +def test_contract_apply_votes_example(tmp_path): + op=dict(op="apply_votes",payload=dict(from_slot_exclusive=0,to_slot_inclusive=3)) + worker,p=setup(tmp_path,[-1,0,1],[op]);worker.handle(req("initialize",p)) + result=worker.handle(req(op["op"],op["payload"],2)) + assert result["observed_state_cursors"]["votes"]["slot"]==3 + assert worker.conv.raw_rating_mat.loc[2,3]==-1 + + +def test_contract_empty_snapshot(tmp_path): + ops=[dict(op="compute",payload=dict(compute_id="c0",logical_clock=0)),dict(op="snapshot",payload=dict(checkpoint_id="checkpoint-000"))] + worker,p=setup(tmp_path,ops=ops);worker.handle(req("initialize",p)) + worker.handle(req("compute",ops[0]["payload"],2)) + response=worker.handle(req("snapshot",ops[1]["payload"],4)) + assert set(response)=={"checkpoint_id","manifest"} + main=json.loads((worker.output_root/"checkpoint-000/main.json").read_text()) + assert main["n"]==0 and main["tids"]==[] and main["pca"]["center"]==[] + assert main["lastVoteTimestamp"]==0 + assert json.loads((worker.output_root/"checkpoint-000/bidtopid.json").read_text())==dict(zid="synthetic-empty",bidToPid=[],lastVoteTimestamp=0) + + +@pytest.mark.parametrize("vote",[None,True,2,"-1"]) +def test_reject_invalid_vote_before_mutation(tmp_path,vote): + worker,p=setup(tmp_path,[vote]) + with pytest.raises(ProtocolError) as error: worker.handle(req("initialize",p)) + assert error.value.code=="INVALID_INPUT" and worker.state=="NEW" + + +@pytest.mark.parametrize("raw",['{"a":1,"a":2}','{"a":NaN}','{"a":Infinity}']) +def test_strict_json(raw): + with pytest.raises(ProtocolError):strict_json(raw) + + +def test_checksum_and_path(tmp_path): + worker,p=setup(tmp_path) + p["input_manifest"]["sha256"]="0"*64 + with pytest.raises(ProtocolError) as error:worker.handle(req("initialize",p)) + assert error.value.code=="CHECKSUM_MISMATCH" + + +def test_symlink(tmp_path): + worker,p=setup(tmp_path) + (worker.input_root/"linked").symlink_to(worker.input_root/"manifest.json") + p["input_manifest"]["path"]="linked" + with pytest.raises(ProtocolError): worker.handle(req("initialize",p)) + + +def test_sequence_replay_rejected(tmp_path): + worker,p=setup(tmp_path) + worker.handle(req("initialize",p)) + with pytest.raises(ProtocolError):worker.handle(req("initialize",p)) + + +def test_gap_rejected(tmp_path): + op=dict(op="apply_votes",payload=dict(from_slot_exclusive=1,to_slot_inclusive=2)) + worker,p=setup(tmp_path,[-1,1],[op]);worker.handle(req("initialize",p)) + with pytest.raises(ProtocolError):worker.handle(req(op["op"],op["payload"],2)) + + +@pytest.mark.parametrize('raw',[b'[]\n',b'{"bad":true}\n',b'NaN\n',b'x'*65537+b'\n']) +def test_stdio_errors_are_terminal_and_framed(tmp_path,raw): + import os,subprocess,sys + env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1') + process=subprocess.run([sys.executable,'-m','polismath.engine_adapter','--input-root',str(tmp_path),'--output-root',str(tmp_path/'out')],input=raw,capture_output=True,env=env,timeout=20) + assert process.returncode==1 + lines=process.stdout.splitlines();assert len(lines)==1 + response=json.loads(lines[0]);assert response['ok'] is False + + +def test_restore_rejects_wrong_parent(tmp_path): + ops=[dict(op='restore',payload=dict(profile='rebuild-prefix/1',parent={'foreign':True},payload={},vote_cursor=0,moderation_cursor=0))] + worker,p=setup(tmp_path,ops=ops);worker.handle(req('initialize',p)) + with pytest.raises(ProtocolError) as error:worker.handle(req('restore',ops[0]['payload'],2)) + assert error.value.code=='STATE_INCOMPATIBLE' + + +@pytest.mark.parametrize("key,value,reason", [ + ("candidate_schema", "polis-input/1", "CANDIDATE_SCHEMA_MISMATCH"), + ("engine_version", "foreign-engine/1", "ENGINE_VERSION_MISMATCH"), + ("input_digest", "0" * 64, "INPUT_DIGEST_MISMATCH"), + ("schedule_digest", "0" * 64, "SCHEDULE_DIGEST_MISMATCH"), + ("operation_id", None, "MALFORMED_CANDIDATE"), + ("engine_version", 1, "MALFORMED_CANDIDATE"), + ("unexpected", "field", "MALFORMED_CANDIDATE"), +]) +def test_candidate_admission_refusals(tmp_path, key, value, reason): + worker, p = setup(tmp_path) + p["admission"][key] = value + with pytest.raises(ProtocolError) as error: + worker.handle(req("initialize", p)) + assert error.value.code == reason + assert worker.state == "NEW" and not hasattr(worker, "conv") + + +@pytest.mark.parametrize("value", [None, [], {}, {"candidate_schema": CANDIDATE_SCHEMA}]) +def test_malformed_candidate_admission(tmp_path, value): + worker, p = setup(tmp_path) + p["admission"] = value + with pytest.raises(ProtocolError) as error: + worker.handle(req("initialize", p)) + assert error.value.code == "MALFORMED_CANDIDATE" + assert not hasattr(worker, "conv") + + +def test_reserved_manifest_schema_rejected_even_with_matching_digest(tmp_path): + worker, p = setup(tmp_path) + path = worker.input_root / "manifest.json" + manifest = json.loads(path.read_text()) + manifest["schema"] = "polis-input/1" + path.write_text(json.dumps(manifest)) + p["input_manifest"] = descriptor(worker.input_root, "manifest.json") + p["admission"]["input_digest"] = p["input_manifest"]["sha256"] + with pytest.raises(ProtocolError) as error: + worker.handle(req("initialize", p)) + assert error.value.code == "CANDIDATE_SCHEMA_MISMATCH" + + +def test_candidate_identity_survives_snapshot(tmp_path): + ops = [dict(op="compute", payload=dict(compute_id="c", logical_clock=0)), + dict(op="snapshot", payload=dict(checkpoint_id="s"))] + worker, p = setup(tmp_path, ops=ops) + initialized = worker.handle(req("initialize", p)) + assert initialized["admission"] == p["admission"] + for rid, op in enumerate(ops, 2): + worker.handle(req(op["op"], op["payload"], rid)) + checkpoint = json.loads((worker.output_root / "s/manifest.json").read_bytes()) + assert checkpoint["admission"] == p["admission"] + assert checkpoint["schema"] == "polis-candidate-checkpoint/1" diff --git a/delphi/tests/coordinator/test_equivalence.py b/delphi/tests/coordinator/test_equivalence.py new file mode 100644 index 0000000000..c674cebbda --- /dev/null +++ b/delphi/tests/coordinator/test_equivalence.py @@ -0,0 +1,208 @@ +import concurrent.futures +import hashlib +import json +import os +import subprocess +import sys +import threading +from pathlib import Path +import pytest +from coordinator.conftest import ROOT, ARTIFACTS, FOLD, MAPPING, seed, connect, rows, assert_coherent +from polismath.replay.crosslang import canonicalize_blob + + +def canonical(tables): + result={} + for table in ("math_main","math_bidtopid","math_ptptstats"): + blob=dict(tables[table]["data"]) + if table=="math_main":blob.pop("math_tick",None) # declared engine wall-clock metadata + result[table]=canonicalize_blob(blob) if table=="math_main" else blob + return result + + +def hash_blob(value): + return hashlib.sha256(json.dumps(value,sort_keys=True,separators=(',',':'),ensure_ascii=False,allow_nan=False).encode()).hexdigest() + + +def diff(a,b,path=""): + out=[] + if isinstance(a,dict) and isinstance(b,dict): + for key in sorted(a.keys()|b.keys()): + if key not in a or key not in b:out.append(dict(path=path+'/'+key,a=a.get(key),b=b.get(key))) + else:out.extend(diff(a[key],b[key],path+'/'+key)) + elif isinstance(a,list) and isinstance(b,list) and len(a)==len(b): + for i,(x,y) in enumerate(zip(a,b)):out.extend(diff(x,y,path+'/'+str(i))) + elif a!=b:out.append(dict(path=path,a=a,b=b)) + return out + + +@pytest.mark.parametrize("fixture",["synthetic","vw","biodiversity"]) +def test_polarity_through_actual_database_ingress(db,launch,fixture): + if fixture=="synthetic":seed(db) + else:seed_vw(db,slug=fixture) + launch(db).done() + a=canonical(rows(db)) + c=connect(db) + with c.cursor() as cur:cur.execute("UPDATE votes SET vote=-vote") + c.close() + launch(db,env="positive",extra={"STORAGE_AGREE_VALUE":"1"}).done() + b=canonical(rows(db,env="positive")) + launch(db,env="negative").done() + negative=canonical(rows(db,env="negative")) + delta=diff(a,b) + (ARTIFACTS/f"polarity-{fixture}.json").write_text(json.dumps({"fixture":fixture,"a":hash_blob(a),"b":hash_blob(b),"negative":hash_blob(negative),"deltas":delta},indent=2)) + assert delta==[] + assert hash_blob(a)==hash_blob(b) + assert hash_blob(a)!=hash_blob(negative) + + +def vw_events(slug="vw"): + from polismath.replay.real_data import load_export_votes + return load_export_votes(slug).votes + + +def seed_vw(db,limit=None,slug="vw"): + events=vw_events(slug) + pids=sorted({v.pid for v in events});tids=sorted({v.tid for v in events}) + seed(db,n_ptpts=0,n_cmts=0,votes=False) + c=connect(db) + with c.cursor() as cur: + cur.execute("SET session_replication_role=replica") + for p in pids:cur.execute("INSERT INTO participants(zid,pid,uid,mod) VALUES(1,%s,%s,0)",(p,100000+p)) + for t in tids:cur.execute("INSERT INTO comments(zid,tid,pid,uid,txt,mod,is_meta,created,modified) VALUES(1,%s,0,100000,%s,0,false,1000,1000)",(t,f'Synthetic comment {t}')) + c.close() + insert_events(db,events[:limit] if limit is not None else events) + return events + + +def insert_events(db,events): + c=connect(db) + with c.cursor() as cur: + cur.executemany("INSERT INTO votes(zid,pid,tid,vote,created) VALUES(1,%s,%s,%s,%s)",[(v.pid,v.tid,-v.sign,v.t_ms) for v in events]) + c.close() + + +def python_checkpoint(db): + env=dict(os.environ,DATABASE_URL=db,PYTHONPATH=str(ROOT/'delphi'),MATH_ENV='python',MATH_PYTHON_ENV='python', + OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',PYTHONDONTWRITEBYTECODE='1') + r=subprocess.run([sys.executable,str(Path(__file__).with_name('reference_child.py'))],env=env,cwd=ROOT/'delphi',capture_output=True,text=True,timeout=150) + assert r.returncode==0,(r.stdout,r.stderr) + + +def test_vw_dual_namespace_live_equivalence(db,launch): + events=seed_vw(db,0) + cuts=[len(events)//4,len(events)//2,len(events)] + inventory=[];previous=0;observer_errors=[];observations=[0];stop=threading.Event() + def observer(): + while not stop.is_set(): + for env in ('python','rustproto'): + snapshot=rows(db,env=env) + if snapshot['math_main']: + if any(snapshot[t] is None for t in snapshot):observer_errors.append('missing table');continue + if len({r['math_tick'] for r in snapshot.values()})!=1:observer_errors.append('mixed ticks') + a,b,p=[snapshot[t]['data'] for t in ('math_main','math_bidtopid','math_ptptstats')] + observer_errors.extend(MAPPING(a,b,p));observations[0]+=1 + stop.wait(.02) + thread=threading.Thread(target=observer);thread.start() + try: + for cut in cuts: + insert_events(db,events[previous:cut]);previous=cut + # Both actual processes operate against one visibility checkpoint. + with concurrent.futures.ThreadPoolExecutor(1) as pool: + reference=pool.submit(python_checkpoint,db) + launch(db).done();reference.result() + a=rows(db);b=rows(db,env='python') + assert a['math_main']['math_tick']==b['math_main']['math_tick'] + ca,cb=canonical(a),canonical(b) + deltas=diff(ca,cb) + fold=FOLD.fold_votes([dict(pid=v.pid,tid=v.tid,vote=-v.sign,created=v.t_ms) for v in events[:cut]]) + fold_errors={env:FOLD.check_published_against_fold(t['math_main']['data'],fold,require_all_clustered=False) for env,t in [('rustproto',a),('python',b)]} + clustered={env:{pid for bucket in t['math_bidtopid']['data']['bidToPid'] for pid in bucket} for env,t in [('rustproto',a),('python',b)]} + for env,tables in [('rustproto',a),('python',b)]: + main=tables['math_main']['data'] + for index,members in enumerate(tables['math_bidtopid']['data']['bidToPid']): + member_set=set(members) + subset=FOLD.fold_votes([dict(pid=v.pid,tid=v.tid,vote=-v.sign,created=v.t_ms) for v in events[:cut] if v.pid in member_set]) + totals=subset.per_comment_totals() + for tid,buckets in main['votes-base'].items(): + expected=totals.get(int(tid),{'A':0,'D':0,'S':0}) + actual={key:buckets[key][index] for key in ('A','D','S')} + if actual!=expected:fold_errors[env].append(dict(tid=tid,bucket=index,actual=actual,expected=expected)) + inventory.append(dict(cut=cut,tick=a['math_main']['math_tick'],rust=hash_blob(ca),python=hash_blob(cb),deltas=deltas,fold_errors=fold_errors, + unclustered_participants={env:len(fold.participants-members) for env,members in clustered.items()}, + fold_scope='all-voter counts/timestamps/events plus independent latest-cell A/D/S per emitted bucket; no independent eligibility certificate')) + finally: + stop.set();thread.join() + (ARTIFACTS/'vw-equivalence.json').write_text(json.dumps(dict(checkpoints=inventory,observer_errors=observer_errors,observations=observations[0],profile='rebuild-prefix at each checkpoint; static unmoderated comment snapshot'),indent=2)) + assert not observer_errors + assert len(inventory)==3 + assert all(not x['deltas'] for x in inventory) + assert all(not e for x in inventory for e in x['fold_errors'].values()) + + +def test_polarity_rebuild_schedule_with_revotes(db,launch): + seed(db,votes=False) + def write(sign,events): + c=connect(db) + with c.cursor() as cur: + cur.execute('DELETE FROM votes WHERE zid=1') + cur.executemany('INSERT INTO votes(zid,pid,tid,vote,created) VALUES(1,%s,%s,%s,%s)',[(p,t,v*sign,stamp) for p,t,v,stamp in events]) + c.close() + events=[(p,t,(-1 if p%2==0 else 1),1000+p*4+t) for p in range(6) for t in range(4)] + variants=[] + for step in range(3): + if step:events.append((0,0,step-1,2000+step)) + write(1,events);launch(db).done();a=canonical(rows(db)) + write(-1,events);launch(db,env='positive',extra={'STORAGE_AGREE_VALUE':'1'}).done();b=canonical(rows(db,env='positive')) + launch(db,env='negative').done();n=canonical(rows(db,env='negative')) + assert hash_blob(a)==hash_blob(b) + assert hash_blob(a)!=hash_blob(n) + variants.append(dict(step=step,positive=hash_blob(a),paired=hash_blob(b),negative=hash_blob(n))) + (ARTIFACTS/'polarity-rebuild-schedule.json').write_text(json.dumps(variants,indent=2)) + + +def test_semantic_tie_key_is_a_declared_contract_term(db,launch): + """Rev5 item 1. Two vote rows share (tid,pid,created) and differ only in the + raw sign, so the published generation is decided by the trailing tie term. + Ordering on the semantic vote (raw x storage_agree_value) keeps the mirrored + conversation identical; ordering on the raw sign would reverse that pair and + break the polarity property. The declaration and its census are recorded.""" + seed(db) + c=connect(db) + with c.cursor() as cur: + cur.execute('SET session_replication_role=replica') + cur.execute('DELETE FROM votes WHERE zid=1 AND pid=0 AND tid=0') + for value in (1,-1): + cur.execute('INSERT INTO votes(zid,pid,tid,vote,created) VALUES(1,0,0,%s,1500)',(value,)) + c.close() + launch(db).done() + a=canonical(rows(db)) + declared=rows(db)['math_ticks']['input_checkpoint']['ordering'] + assert declared['schema']=='polis-order/1' + assert declared['semantic_vote']=='raw_vote * storage_agree_value' + assert declared['storage_agree_value']==-1 and declared['algorithm_digest'] + assert [t['name'] for t in declared['terms']]==['tid','pid','created_ms','semantic_vote','weight_x_32767'] + assert declared['terms'][-1]['nulls']=='first' + census=declared['equal_time_census'] + assert census['tied_groups']==1 and census['tied_rows']==2 + assert census['key']==['tid','pid','created_ms'] + assert census['resolved_by']==['semantic_vote','weight_x_32767'] + c=connect(db) + with c.cursor() as cur:cur.execute('UPDATE votes SET vote=-vote WHERE zid=1') + c.close() + launch(db,env='positive',extra={'STORAGE_AGREE_VALUE':'1'}).done() + b=canonical(rows(db,env='positive')) + launch(db,env='negative').done() + negative=canonical(rows(db,env='negative')) + mirrored=rows(db,env='positive')['math_ticks']['input_checkpoint']['ordering'] + assert mirrored['storage_agree_value']==1 + # The pinned ordering algorithm is polarity-bound, so the two conventions + # cannot silently share one declaration. + assert mirrored['algorithm_digest']!=declared['algorithm_digest'] + assert mirrored['equal_time_census']==census + (ARTIFACTS/'semantic-tie-key.json').write_text(json.dumps(dict( + declared=declared,mirrored_digest=mirrored['algorithm_digest'], + a=hash_blob(a),b=hash_blob(b),negative=hash_blob(negative),deltas=diff(a,b)),indent=2)) + assert diff(a,b)==[] + assert hash_blob(a)==hash_blob(b) + assert hash_blob(a)!=hash_blob(negative) diff --git a/delphi/tests/coordinator/test_incremental.py b/delphi/tests/coordinator/test_incremental.py new file mode 100644 index 0000000000..4e6d06b845 --- /dev/null +++ b/delphi/tests/coordinator/test_incremental.py @@ -0,0 +1,200 @@ +"""CO01 incremental discovery and the metrics the contract names. + +The probe is a filter, never a gate: a differing probe always forces the +authoritative snapshot, and a matching probe only skips a read while the +conversation's last full reconciliation is younger than the configured ceiling. +`test_the_aggregate_probe_is_weak_but_the_ceiling_repairs_it` is the negative +control for exactly that, staged with a change no aggregate can see. +""" +import json +import subprocess +import time +from pathlib import Path + +from coordinator.conftest import BINARY, assert_coherent, connect, rows, seed, wait + + +def query(db, sql, args=()): + c = connect(db) + try: + with c.cursor() as cur: + cur.execute(sql, args) + return cur.fetchall() if cur.description else None + finally: + c.close() + + +def emf(path, operation=None): + """Every Embedded Metric Format record the process wrote, in order.""" + path = Path(path) + if not path.exists(): + return [] + out = [] + for line in path.read_text().splitlines(): + if not line.strip(): + continue + record = json.loads(line) + if operation is None or record.get("operation") == operation: + out.append(record) + return out + + +def catalog(): + return json.loads(subprocess.check_output([str(BINARY), "metrics"], text=True)) + + +def test_probe_skips_the_full_source_read_and_a_real_change_still_publishes(db, launch, tmp_path): + seed(db) + first, second, third = (tmp_path / f"{n}.jsonl" for n in ("first", "second", "third")) + + launch(db, extra={"P026_METRICS": str(first)}).done() + pass_one = emf(first, "source_pass")[-1] + assert pass_one["SourcePassReconciled"] == 1 + assert pass_one["SourcePassPublished"] == 1 + assert pass_one["SourcePassSkipped"] == 0 + assert_coherent(db) + published = rows(db)["math_main"]["math_tick"] + + # Nothing changed: the probe matches the one recorded before the snapshot + # that certified this generation, so no source read is taken at all. + launch(db, extra={"P026_METRICS": str(second)}).done() + pass_two = emf(second, "source_pass")[-1] + assert pass_two["SourcePassProbed"] == 1 + assert pass_two["SourcePassSkipped"] == 1 + assert pass_two["SourcePassReconciled"] == 0 + assert emf(second, "reconciliation") == [], "a skipped conversation must not read the source" + assert rows(db)["math_main"]["math_tick"] == published + + # A committed vote changes the probe, so the full authoritative snapshot is + # taken and the new generation is published. + query(db, "INSERT INTO votes(zid,pid,tid,vote,created) VALUES(1,0,0,1,2000)") + launch(db, extra={"P026_METRICS": str(third)}).done() + pass_three = emf(third, "source_pass")[-1] + assert pass_three["SourcePassSkipped"] == 0 + assert pass_three["SourcePassReconciled"] == 1 + assert pass_three["SourcePassPublished"] == 1 + tables = assert_coherent(db) + assert tables["math_main"]["math_tick"] > published + assert tables["math_ticks"]["input_checkpoint"]["event_count"] == 25 + + +def test_the_aggregate_probe_is_weak_but_the_reconciliation_ceiling_repairs_it(db, launch, tmp_path): + """Rev5: count/max are hints, not completeness proof. Stage a change every + aggregate in the probe is blind to, show the fast path really does miss it, + and show the bounded authoritative reconciliation finds it anyway.""" + seed(db) + launch(db).done() + before = rows(db)["math_main"] + # Swap two votes' values: count, min/max/sum(created), sum(vote), sum(weight), + # comment and participant aggregates are all identical afterwards. + query(db, "UPDATE votes SET vote=1 WHERE zid=1 AND pid=0 AND tid=0") + query(db, "UPDATE votes SET vote=-1 WHERE zid=1 AND pid=0 AND tid=1") + hidden = tmp_path / "hidden.jsonl" + launch(db, extra={"P026_METRICS": str(hidden), "P026_RECONCILE_SECONDS": "3600"}).done() + skipped = emf(hidden, "source_pass")[-1] + assert skipped["SourcePassSkipped"] == 1, "the probe genuinely cannot see this change" + assert rows(db)["math_main"]["math_tick"] == before["math_tick"] + + # ... which is why the probe may never be the only rebuild gate. + time.sleep(1.2) + repaired = tmp_path / "repaired.jsonl" + launch(db, extra={"P026_METRICS": str(repaired), "P026_RECONCILE_SECONDS": "1"}).done() + pass_two = emf(repaired, "source_pass")[-1] + assert pass_two["SourcePassSkipped"] == 0 + assert pass_two["SourcePassReconciled"] == 1 + assert pass_two["SourcePassPublished"] == 1 + tables = assert_coherent(db) + assert tables["math_main"]["math_tick"] > before["math_tick"] + assert tables["math_main"]["data"] != before["data"] + + +def test_disabling_the_fast_path_restores_the_unconditional_full_sweep(db, launch, tmp_path): + """CO05: the complete pass stays available as the fallback.""" + seed(db) + launch(db).done() + metrics = tmp_path / "full.jsonl" + launch(db, extra={"P026_METRICS": str(metrics), "P026_INCREMENTAL": "0"}).done() + record = emf(metrics, "source_pass")[-1] + assert record["SourcePassSkipped"] == 0 + assert record["SourcePassReconciled"] == 1, "the full snapshot runs regardless of the probe" + assert record["SourcePassPublished"] == 0, "and it agrees with the published generation" + assert record["context"]["incremental"] is False + + +def test_emitted_records_match_the_declared_catalog_and_p031_dimensions(db, launch, tmp_path): + declared = catalog() + assert declared["namespace"] == "Polis/Math" + assert declared["dimensions"] == ["Environment", "MathEnv"] + names = {m["name"]: m for m in declared["metrics"]} + seed(db) + metrics = tmp_path / "m.jsonl" + launch(db, extra={"P026_METRICS": str(metrics)}).done() + records = emf(metrics) + assert records + for record in records: + block = record["_aws"]["CloudWatchMetrics"][0] + assert block["Namespace"] == "Polis/Math" + # P-031: "Math uses only fixed Environment=prod, MathEnv=prod ... No + # conversation/job/report/run/instance dimensions." + assert block["Dimensions"] == [["Environment", "MathEnv"]] + assert record["Environment"] == "synthetic" and record["MathEnv"] == "rustproto" + assert isinstance(record["_aws"]["Timestamp"], int) + for entry in block["Metrics"]: + assert entry["Name"] in names, entry + assert entry["Unit"] == names[entry["Name"]]["unit"] + assert isinstance(record[entry["Name"]], (int, float)) + pass_record = emf(metrics, "source_pass")[-1] + assert pass_record["SourcePassHealthy"] == 1 + assert pass_record["LeaseAcquired"] == 1 + assert pass_record["PublishCommitted"] == 1 + assert pass_record["LeaseFenced"] == 0 and pass_record["PublishRefused"] == 0 + assert pass_record["MetricsDropped"] == 0 + for gauge in ("OldestReconciliationAgeSeconds", "ReconciliationBacklogConversations", + "FailureBacklogConversations", "OldestUnrepairedAgeSeconds"): + assert gauge in pass_record, gauge + per_zid = emf(metrics, "reconciliation")[-1] + for timing in ("SourceReadSeconds", "ComputeSeconds", "PublishSeconds"): + assert per_zid[timing] >= 0 + assert per_zid["context"]["zid"] == 1 + assert emf(metrics, "conversation")[-1]["ConversationLatencySeconds"] > 0 + + +def test_scan_age_and_backlog_are_exposed_and_drain(db, launch, tmp_path): + """CO01 asks for scan age and backlog. One conversation per page, so the + first pass leaves the second conversation measurably unreconciled.""" + seed(db, 1) + seed(db, 2) + metrics = tmp_path / "backlog.jsonl" + child = launch(db, "run", extra={"P026_METRICS": str(metrics), "P026_PAGE_SIZE": "1", + "P026_GAUGE_SECONDS": "0", "P026_POLL_MS": "50"}) + wait(lambda: any(r.get("ReconciliationBacklogConversations") == 1 + for r in emf(metrics, "source_pass")), + alive=child, why="the unvisited conversation is counted as backlog") + wait(lambda: any(r.get("ReconciliationBacklogConversations") == 0 + for r in emf(metrics, "source_pass")), + alive=child, why="the backlog drains once every conversation is reconciled") + child.kill() + assert_coherent(db, 1) + assert_coherent(db, 2) + ages = [r["OldestReconciliationAgeSeconds"] for r in emf(metrics, "source_pass") + if "OldestReconciliationAgeSeconds" in r] + assert ages and all(age >= 0 for age in ages) + + +def test_failures_appear_as_backlog_and_unrepaired_age_without_faking_health(db, launch, tmp_path): + """CO06 oldest unrepaired age. A stuck conversation must be visible as + stuck; the pass itself still completed, and SourcePassHealthy says only that.""" + seed(db) + metrics = tmp_path / "failures.jsonl" + child = launch(db, "run", extra={"P026_METRICS": str(metrics), + "P026_PYTHON": str(tmp_path / "no-such-interpreter"), + "P026_GAUGE_SECONDS": "0", "P026_POLL_MS": "50"}) + wait(lambda: any(r.get("FailureBacklogConversations", 0) >= 1 + for r in emf(metrics, "source_pass")), + alive=child, why="the failing conversation reaches the durable failure backlog") + child.kill() + stuck = [r for r in emf(metrics, "source_pass") if r.get("FailureBacklogConversations", 0) >= 1][-1] + assert stuck["SourcePassDeferred"] >= 1 + assert stuck["OldestUnrepairedAgeSeconds"] >= 0 + assert stuck["SourcePassHealthy"] == 1, "the loop is alive; stuck work is a different signal" + assert rows(db)["math_main"] is None, "nothing was published" diff --git a/delphi/tests/coordinator/test_node_reader.py b/delphi/tests/coordinator/test_node_reader.py new file mode 100644 index 0000000000..fd1b8795ea --- /dev/null +++ b/delphi/tests/coordinator/test_node_reader.py @@ -0,0 +1,197 @@ +"""CO08 / D4: the bytes the REAL Node server serves, from both writers. + +The Rust coordinator and the pinned Python reference writer publish the same +checkpoint into two namespaces of one database. The actual server modules — +`server/src/utils/pca.ts` and `server/src/utils/participants.ts` — are then +loaded in one Node process and asked for both namespaces, and the served +artefacts are compared byte for byte: + + * `asJSON`, the string `participationInit` embeds as `response.pca`, + * `asBufferOfGzippedJson`, the literal body of `GET /api/v3/math/pca2`, + * the `_.pick(asPOJO, keys)` projection that route serves with `?keys=`, + * `getBidIndexToPidMapping`, and the `getPidsForGid` join over both. + +Scope, stated honestly: there is no `loadBundle` in the server today — #2703 +scoped the existing reader by `[math_env, zid]`, it did not introduce CO04's +Bundle. This exercises the consumer that exists, in-process rather than over +HTTP, and it does not close the CO04 Node cache-unit rewrite or the private +2,884-case served corpus. +""" +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest +from coordinator.conftest import ROOT, assert_coherent, connect, rows, seed +from coordinator.test_equivalence import python_checkpoint + +HARNESS = ROOT / "coordinator-rs/tools/node_reader.cjs" +EVIDENCE = ROOT / "coordinator-rs/evidence" +SERVER_MODULES = ROOT / "server/node_modules" +COMPARED = ("asJSON_sha256", "asJSON_bytes", "gzip_sha256", "gzip_bytes", + "keys_projection_sha256", "tids", "n", "repness_keys", "consensus_shape", + "mapping_sha256", "bid_to_pid", "pids_for_gid", "mapping_is_error") + +requires_server_modules = pytest.mark.skipif( + not SERVER_MODULES.exists(), + reason="server/node_modules is absent; install or link it to run the real Node reader", +) + + +def node_read(db, zid=1, envs=("python", "rustproto"), keys=("tids", "n", "repness"), gids=(0, 1)): + spec = json.dumps({"zid": zid, "envs": list(envs), "keys": list(keys), "gids": list(gids)}) + result = subprocess.run( + ["node", str(HARNESS), spec], + env=dict(os.environ, DATABASE_URL=db, MATH_ENV="rustproto", + CACHE_MATH_RESULTS="true", NODE_ENV="test"), + cwd=ROOT / "server", capture_output=True, text=True, timeout=300, + ) + assert result.returncode == 0, (result.stdout, result.stderr) + return {ns["math_env"]: ns for ns in json.loads(result.stdout)["namespaces"]} + + +def rust_and_python_publish(db, launch): + """One visibility checkpoint, both actual writers, two namespaces — plus a + deliberately different third namespace, so that "the two agree" is only + evidence once "and a different one does not" is also true (Rev7).""" + launch(db).done() + python_checkpoint(db) + # Same source, opposite declared agree convention: a genuinely different + # generation for the same zid, in its own namespace. + launch(db, env="positive", extra={"STORAGE_AGREE_VALUE": "1"}).done() + a, b = rows(db, env="rustproto"), rows(db, env="python") + assert a["math_main"] is not None and b["math_main"] is not None + assert a["math_main"]["math_tick"] == b["math_main"]["math_tick"] + return a, b + + +@requires_server_modules +def test_real_node_reader_serves_identical_bytes_for_both_writers(db, launch): + seed(db) + rust_and_python_publish(db, launch) + # A second checkpoint. This harness calls `getPca(zid, undefined)`, whose + # cold behaviour at a committed generation of zero is a documented reader + # asymmetry (test_step2_review_controls.py shows the real route serving + # generation zero with 200 and ETag "0"), so the comparison uses tick 1. + # Both namespaces advance together. + c = connect(db) + with c.cursor() as cur: + cur.execute("INSERT INTO votes(zid,pid,tid,vote,created) VALUES(1,0,0,1,2000)") + c.close() + tables, _ = rust_and_python_publish(db, launch) + assert tables["math_main"]["math_tick"] == 1 + assert_coherent(db, env="rustproto") + + served = node_read(db) + assert set(served) == {"python", "rustproto"} + for namespace in served.values(): + assert namespace["present"], namespace + assert namespace["mapping_is_error"] is False + assert namespace["etag_math_tick"] == 1 + differences = {field: (served["rustproto"][field], served["python"][field]) + for field in COMPARED + if served["rustproto"][field] != served["python"][field]} + (EVIDENCE / "d4-node-reader.json").write_text(json.dumps({ + "profile": "real server/src/utils/pca.ts + participants.ts, in-process, one Node process", + "zid": 1, "math_tick": 1, "compared_fields": list(COMPARED), + "differences": differences, + "served": {env: {k: v for k, v in ns.items() if k in COMPARED or k == "etag_math_tick"} + for env, ns in served.items()}, + }, indent=2, sort_keys=True)) + assert differences == {}, differences + # The join is real, not empty: at least one group maps to participants. + joins = served["rustproto"]["pids_for_gid"] + assert any(isinstance(v, list) and v for v in joins.values()), joins + + # Rev7: two namespaces agreeing does not prove namespace isolation unless a + # deliberately different third namespace is served differently. Publish the + # same conversation under the opposite agree convention, which produces a + # genuinely different generation, and require the reader to tell them apart. + both = node_read(db, envs=("rustproto", "positive", "reader")) + foreign = both["positive"] + assert foreign["present"], foreign + assert foreign["asJSON_sha256"] != served["rustproto"]["asJSON_sha256"] + assert foreign["gzip_sha256"] != served["rustproto"]["gzip_sha256"] + assert foreign["keys_projection_sha256"] != served["rustproto"]["keys_projection_sha256"] + # The mirrored convention keeps the same clustering, so bidToPid is + # legitimately identical; recording that rather than asserting a difference + # that does not exist. + assert foreign["mapping_sha256"] == served["rustproto"]["mapping_sha256"] + # A namespace with no math rows for the same zid: the mapping query really + # is scoped by math_env, so it finds nothing while the populated one does. + assert both["reader"]["mapping_is_error"] is True + assert served["rustproto"]["mapping_is_error"] is False + + +@requires_server_modules +def test_published_empty_math_versus_the_servers_own_empty_presentation(db, launch): + """D4's zero-vote-with-comments shape, and a C7 observation from it. + + The pinned #2704 Python poller publishes **nothing** for a conversation + with approved comments and no votes: its vote watermark finds no work. So + there is no two-writer equality to assert here. What the real reader does + instead is synthesize `createEmptyPcaStructure` for the namespace with no + row, while serving the coordinator's explicitly published empty generation + for the namespace that has one. + + Both are recorded, and the stable presentation fields are asserted equal. + Byte equality between a published empty generation and the server's own + synthesized empty presentation is NOT claimed: the full difference list is + written to evidence for polis-empty-served/1 to rule on. + """ + seed(db, votes=False) + c = connect(db) + with c.cursor() as cur: + cur.execute("UPDATE comments SET mod=1 WHERE zid=1 AND tid<2") + c.close() + launch(db).done() + python_checkpoint(db) + assert rows(db, env="python")["math_main"] is None, ( + "the reference writer publishes no generation for a zero-vote conversation") + # A second generation, for the same reason as above: this harness's + # `getPca(zid, undefined)` caller does not return a cold generation zero. + c = connect(db) + with c.cursor() as cur: + cur.execute("UPDATE comments SET mod=1 WHERE zid=1") + c.close() + launch(db).done() + assert rows(db, env="rustproto")["math_main"]["math_tick"] == 1 + + served = node_read(db, gids=(0,)) + published, synthesized = served["rustproto"], served["python"] + assert published["present"], published + assert synthesized["present"], "the server synthesizes an empty presentation" + differences = {field: (published[field], synthesized[field]) + for field in COMPARED + if published.get(field) != synthesized.get(field)} + (EVIDENCE / "d4-node-reader-empty.json").write_text(json.dumps({ + "profile": "zero votes, four approved comments; rustproto has a published " + "empty generation, python has none and is synthesized by the server", + "reference_published": False, + "note": "the synthesized empty presentation stamps lastVoteTimestamp with " + "Date.now(), so it is not reproducible across servers or requests; " + "the published empty generation is", + "differences": differences, "served": served}, indent=2, sort_keys=True)) + assert published["n"] == synthesized["n"] == 0 + assert published["consensus_shape"] == synthesized["consensus_shape"] + assert published["mapping_is_error"] is False + # The mapping is the one place the synthesized path has nothing to serve. + assert synthesized["mapping_is_error"] is True + # C7, observed rather than argued: the published math blob lists no + # comments, while the server's own empty presentation lists the approved + # ones straight from `comments`. The contract's ruling is that the server + # must own that listing; this is the two behaviours side by side. + assert published["tids"] == [] + assert synthesized["tids"] == [0, 1, 2, 3] + assert "tids" in differences + # And the synthesized presentation is not even reproducible: pca.ts's + # createEmptyPcaStructure stamps `lastVoteTimestamp: Date.now()`, so two + # servers answering the same request for the same conversation serve + # different bytes. The published generation carries the engine's value. + now = time.time() * 1000 + assert abs(synthesized["last_vote_timestamp"] - now) < 600_000, ( + synthesized["last_vote_timestamp"], now) + assert published["last_vote_timestamp"] != synthesized["last_vote_timestamp"] diff --git a/delphi/tests/coordinator/test_recorder_controls.py b/delphi/tests/coordinator/test_recorder_controls.py new file mode 100644 index 0000000000..2d31dbaf49 --- /dev/null +++ b/delphi/tests/coordinator/test_recorder_controls.py @@ -0,0 +1,85 @@ +"""Controls on the S1 recorder's cargo transcript reader. + +The recorder derives its Rust counts from the run instead of asserting a literal +total. Deriving is not trusting: a transcript carries one `test result:` line per +suite, so a passing suite followed by a failing one still leaves passing lines +behind, and a total assembled from those lines alone would certify a failed run. +These controls pin the refusals. They touch no database, no cargo and no engine — +only synthetic transcripts and status files in a tmp directory. +""" +import importlib.util +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[3] +RECORDER = ROOT / "coordinator-rs/tools/record_s1.py" + +_spec = importlib.util.spec_from_file_location("record_s1", RECORDER) +record_s1 = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(record_s1) + +NAMED = "uncertain_commit_requires_own_epoch_even_at_identical_tick_and_checkpoint ... ok\n" +PASS_ONE = "test result: ok. 12 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out\n" +PASS_TWO = "test result: ok. 19 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out\n" +FAILED = "test result: FAILED. 18 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out\nerror: test failed\n" +IGNORED = "test result: ok. 18 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out\n" + + +def write(artifacts, transcript, status="0"): + for profile in record_s1.CARGO_PROFILES: + (artifacts / f"s1-cargo-{profile}.log").write_text(transcript) + (artifacts / f"s1-cargo-{profile}.status").write_text(status + "\n") + return artifacts + + +def test_multi_suite_success_is_counted_across_every_suite(tmp_path): + """Positive control: several clean suites sum, with no fixed total anywhere.""" + write(tmp_path, NAMED + PASS_ONE + PASS_TWO) + assert record_s1.rust_test_counts(tmp_path) == {"default": 31, "fault": 31} + + +def test_passing_suite_followed_by_failed_suite_is_refused(tmp_path): + """The reviewed defect: the passing lines must not be reported on their own.""" + write(tmp_path, NAMED + PASS_ONE + FAILED) + with pytest.raises(AssertionError, match="failed cargo suite"): + record_s1.rust_test_counts(tmp_path) + + +def test_ignored_tests_are_not_counted_as_passes(tmp_path): + write(tmp_path, NAMED + PASS_ONE + IGNORED) + with pytest.raises(AssertionError, match="ignored cargo tests"): + record_s1.rust_test_counts(tmp_path) + + +def test_nonzero_cargo_exit_status_is_refused(tmp_path): + """A transcript that parses clean cannot rescue a process that exited non-zero.""" + write(tmp_path, NAMED + PASS_ONE, status="101") + with pytest.raises(AssertionError, match="exited 101"): + record_s1.rust_test_counts(tmp_path) + + +def test_missing_cargo_exit_status_is_refused(tmp_path): + write(tmp_path, NAMED + PASS_ONE) + (tmp_path / "s1-cargo-fault.status").unlink() + with pytest.raises(AssertionError, match="no captured exit status"): + record_s1.rust_test_counts(tmp_path) + + +def test_malformed_result_line_is_refused(tmp_path): + write(tmp_path, NAMED + PASS_ONE + "test result: ok. 3 passed\n") + with pytest.raises(AssertionError, match="malformed cargo test result"): + record_s1.rust_test_counts(tmp_path) + + +def test_empty_transcript_is_refused(tmp_path): + write(tmp_path, NAMED) + with pytest.raises(AssertionError, match="no cargo test result line"): + record_s1.rust_test_counts(tmp_path) + + +def test_profiles_must_agree_and_be_nonempty(tmp_path): + write(tmp_path, NAMED + PASS_ONE) + (tmp_path / "s1-cargo-fault.log").write_text(NAMED + PASS_TWO) + with pytest.raises(AssertionError): + record_s1.rust_test_counts(tmp_path) diff --git a/delphi/tests/coordinator/test_recovery.py b/delphi/tests/coordinator/test_recovery.py new file mode 100644 index 0000000000..449d395393 --- /dev/null +++ b/delphi/tests/coordinator/test_recovery.py @@ -0,0 +1,453 @@ +"""Black-box ports of R05/R07/R08/R09/R11/R12; no candidate monkeypatches.""" +import json +import signal +import threading +import time +import pytest +from coordinator.conftest import (ARTIFACTS, FOLD, MAPPING, seed, rows, connect, assert_coherent, expire, + lease, wait, repair_after_unclean_death) + +PUBLISH_STAGES=["after_lease","after_source_selection","after_input_checkpoint","before_worker_apply", + "after_worker_compute","before_ticks","after_ticks","before_bidtopid","after_bidtopid", + "before_ptptstats","after_ptptstats","before_main","after_main","before_commit","after_commit","after_ack"] + + +def test_smoke(db,launch): + seed(db) + launch(db).done() + assert_coherent(db) + + +@pytest.mark.parametrize("stage",PUBLISH_STAGES) +def test_r05_unclean_death_restart_repairs(db,launch,tmp_path,stage): + """R05 with no expire() anywhere: the dead owner's lease is left exactly as + an unclean death leaves it, and the restart has to repair through it.""" + seed(db) + child=launch(db,stage=stage,directory=tmp_path/"latch",extra={"P026_LEASE_SECONDS":"2"}) + ack=child.ack() + child.kill() + held=lease(db) + assert held is not None and held["owner_epoch"]==1 # killed owners never release + interim=rows(db) + if stage not in ("after_commit","after_ack"): + assert all(v is None for v in interim.values()) + else: + assert_coherent(db) + _,err=repair_after_unclean_death(launch,db, + lambda: rows(db)["math_main"] is not None and lease(db)["owner_epoch"]>1, + why=f"repair after unclean death at {stage}") + assert "FENCED" not in err # a dead owner is not a superseding one + tables=assert_coherent(db) + (ARTIFACTS/f"stage-{stage}.json").write_text(json.dumps({"stage":stage,"ack":ack, + "signal":"SIGKILL","exit":-signal.SIGKILL,"restart":"passed","manual_lease_expiry":False, + "repaired_epoch":lease(db)["owner_epoch"],"math_tick":tables["math_main"]["math_tick"], + "postcondition":"coherent; independent fold exact; dead owner's lease never expired by the harness"},indent=2)) + + +def test_r05_restart_inside_a_live_lease_defers_then_repairs(db,launch,tmp_path): + """The liveness defect itself: a restart inside the dead owner's lease + window must defer the zid and stay alive, not refuse and exit.""" + seed(db) + child=launch(db,stage="after_worker_compute",directory=tmp_path/"live",extra={"P026_LEASE_SECONDS":"8"}) + child.ack();child.kill() + held=lease(db) + assert held["unexpired"] and held["owner_epoch"]==1 + restart=launch(db,"run",extra={"P026_LEASE_SECONDS":"8","P026_POLL_MS":"50"}) + for _ in range(20): # inside the window: alive, and publishing nothing + assert restart.proc.poll() is None + assert rows(db)["math_main"] is None + time.sleep(.05) + wait(lambda: rows(db)["math_main"] is not None,timeout=90,alive=restart,why="repair after genuine expiry") + assert_coherent(db) + _,err=restart.kill() + assert "LEASE-UNAVAILABLE" in err and "FENCED" not in err + # The takeover is a strictly greater epoch, never a reused one; the daemon + # keeps cycling after the repair, so only monotonicity is asserted. + assert lease(db)["owner_epoch"]>=2 + + +@pytest.mark.parametrize("stage",["after_worker_compute","before_main"]) +def test_r05_expired_lease_restart_repairs(db,launch,tmp_path,stage): + """Explicit DB-time lease-loss control, kept alongside the unclean-death + matrix: with the lease already gone the restart takes over immediately.""" + seed(db) + child=launch(db,stage=stage,directory=tmp_path/"expired") + child.ack();child.kill() + expire(db) + assert lease(db)["unexpired"] is False + launch(db).done() + assert_coherent(db) + + +def test_r06_cache_eviction_contends_with_same_zid_update(db,launch,tmp_path): + """CO07 R06: an LRU eviction and an update to the very zid being evicted. + No lost input, no dropped conversation, and the warm path must publish what + a cold rebuild of the same source publishes.""" + seed(db,1);seed(db,2) + launch(db).done() + launch(db,env="recovery",extra={"P026_CACHE_CAP":"0"}).done() # cold reference namespace + # P026_RECONCILE_SECONDS=1 so the authoritative path runs every second and + # actually loads bundles into the cache: the incremental fast path skips + # both the source read and the store read, so a cache with nothing resident + # would never fill and never evict. + child=launch(db,"run",stage="cache_eviction_contends_with_same_zid_update", + directory=tmp_path/"cache",extra={"P026_CACHE_CAP":"1","P026_POLL_MS":"20", + "P026_RECONCILE_SECONDS":"1"}) + ack=child.ack() + context=ack["context"] + # the race is staged, not assumed: this zid really is being evicted now + assert context["evicted_zid"]==1 and context["inserted_zid"]==2 + assert context["capacity"]==1 and context["cache_len"]==1 + c=connect(db) + with c.cursor() as cur:cur.execute("INSERT INTO votes(zid,pid,tid,vote,created) VALUES(1,0,0,1,2000)") + c.close() + child.release() + wait(lambda: rows(db)["math_ticks"]["input_checkpoint"]["event_count"]==25, + timeout=90,alive=child,why="the update to the evicted zid is published") + child.kill() + warm=assert_coherent(db,1);assert_coherent(db,2) + launch(db,env="recovery",extra={"P026_CACHE_CAP":"0"}).done() + cold=rows(db,1,env="recovery")["math_main"]["data"] + drop=lambda blob:{k:v for k,v in blob.items() if k!="math_tick"} + assert drop(warm["math_main"]["data"])==drop(cold) # evicted-vs-cold reference + (ARTIFACTS/"stage-cache_eviction_contends_with_same_zid_update.json").write_text(json.dumps( + {"stage":"cache_eviction_contends_with_same_zid_update","ack":ack, + "action":"same-zid vote committed while the LRU eviction of that zid is blocked", + "postcondition":"no lost input; evicted zid republished coherently; warm output equals the cold-rebuild reference"},indent=2)) + + +def test_r07_duplicate_refused_healthy_winner(db,launch,tmp_path): + seed(db) + winner=launch(db,stage="after_lease",directory=tmp_path/"owned") + winner.ack() + loser=launch(db) + # A live owner is LEASE-UNAVAILABLE (recoverable), never FENCED. + _,err=loser.done(code=4) + assert "LEASE-UNAVAILABLE" in err and "FENCED" not in err + assert winner.proc.poll() is None + winner.release();winner.done() + assert_coherent(db) + + +def test_r07_stale_owner_cannot_publish_after_transfer(db,launch,tmp_path): + seed(db) + old=launch(db,stage="after_worker_compute",directory=tmp_path/"old") + old.ack() + expire(db) + launch(db).done() + before=rows(db) + old.release() + _,err=old.done(code=3) + assert "FENCED" in err # superseded owner/epoch, not a merely unavailable lease + assert rows(db)==before + assert_coherent(db) + + +@pytest.mark.parametrize("created",[1023,1010]) +def test_r08_late_and_equal_commit(db,launch,created): + seed(db) + launch(db).done() + c=connect(db) + with c.cursor() as cur: + cur.execute("INSERT INTO votes(zid,pid,tid,vote,created) VALUES(1,0,0,1,%s)",(created,)) + cur.execute("SELECT count(*) FROM votes WHERE zid=1 AND pid=0 AND tid=0 AND vote=1 AND created=%s",(created,)) + assert cur.fetchone()[0]==1 # preserved committed-row control + c.close() + launch(db).done() + assert_coherent(db) + + +def test_r08_moderation_and_unmoderation_without_votes(db,launch): + seed(db);launch(db).done() + c=connect(db) + for value in (-1,0): + with c.cursor() as cur: + cur.execute("SET session_replication_role=replica") + cur.execute("UPDATE comments SET mod=%s,modified=1000 WHERE zid=1 AND tid=1",(value,)) + launch(db).done() + data=rows(db)["math_main"]["data"] + assert set(data["moderation"]["mod_out_tids"])==({1} if value==-1 else set()) + c.close() + + +def test_r09_snapshot_reader_during_actual_writer(db,launch,tmp_path): + seed(db);launch(db).done() + c=connect(db) + with c.cursor() as cur:cur.execute("INSERT INTO votes(zid,pid,tid,vote,created) VALUES(1,0,0,1,2000)") + c.close() + writer=launch(db,stage="after_bidtopid",directory=tmp_path/"writer") + writer.ack() + old=rows(db) + for _ in range(20): + snapshot=rows(db) + assert snapshot==old + a,b,p=[snapshot[t]["data"] for t in ("math_main","math_bidtopid","math_ptptstats")] + assert MAPPING(a,b,p)==[] + writer.release();writer.done() + assert_coherent(db) + assert rows(db)["math_main"]["math_tick"]==old["math_main"]["math_tick"]+1 + + +def test_r09_quiet_incomplete_repair(db,launch): + seed(db);launch(db).done() + c=connect(db) + with c.cursor() as cur:cur.execute("DELETE FROM math_bidtopid WHERE zid=1 AND math_env='rustproto'") + c.close() + launch(db).done() + assert_coherent(db) + + +def test_r11_namespace_restore_and_repair_isolation(db,launch): + seed(db);launch(db,env="python").done() + before=rows(db,env="python") + launch(db).done() + assert rows(db,env="python")==before + c=connect(db) + with c.cursor() as cur:cur.execute("DELETE FROM math_ptptstats WHERE math_env='rustproto'") + c.close();launch(db).done() + assert_coherent(db) + assert rows(db,env="python")==before + + +def test_r12_out_of_order_outside_window_metadata_sweep(db,launch,tmp_path): + seed(db,1);seed(db,2);seed(db,3);seed(db,4) + a=launch(db,stage="after_main",directory=tmp_path/"a",extra={"POLL_ALLOWLIST":"1"}) + a.ack() + launch(db,extra={"POLL_ALLOWLIST":"2,3,4"}).done() + c=connect(db) + with c.cursor() as cur: + cur.execute("SELECT max(caching_tick) FROM math_main WHERE math_env='rustproto'") + high=cur.fetchone()[0] + cur.execute("SELECT zid FROM math_main WHERE caching_tick>0 AND math_env='rustproto'") + seen={r[0] for r in cur} + a.release();a.done() + with c.cursor() as cur: + cur.execute("SELECT zid FROM math_main WHERE caching_tick>%s AND math_env='rustproto'",(high,)) + seen.update(r[0] for r in cur) + assert seen=={2,3,4} # naive cursor MUST still miss the late committer + out,_=launch(db,"poll",args=(high,)).done() + assert 1 not in {r["zid"] for r in json.loads(out)} # beyond finite W + swept=set();after=0 + for _ in range(3): + out,_=launch(db,"scan",args=(after,)).done() + page=json.loads(out) + swept.update(r["zid"] for r in page) + if page:after=page[-1]["zid"] + assert swept=={1,2,3,4} + c.close() + + +def test_r12_sequence_negative_control_seen_two(db,launch,tmp_path): + seed(db,1);seed(db,2) + a=launch(db,stage="after_main",directory=tmp_path/"a",extra={"POLL_ALLOWLIST":"1"}) + a.ack();launch(db,extra={"POLL_ALLOWLIST":"2"}).done() + c=connect(db) + with c.cursor() as cur: + cur.execute("SELECT zid,caching_tick FROM math_main WHERE math_env='rustproto'") + observed=cur.fetchall();seen={r[0] for r in observed};high=max(r[1] for r in observed) + a.release();a.done() + for _ in range(3): + with c.cursor() as cur: + cur.execute("SELECT zid FROM math_main WHERE caching_tick>%s AND math_env='rustproto'",(high,)) + seen.update(r[0] for r in cur) + assert seen=={2} + assert rows(db,1)["math_main"]["caching_tick"]!=rows(db,2)["math_main"]["caching_tick"] + c.close() + + +def test_unmarked_kill_is_unclean(db,launch): + seed(db) + child=launch(db,mode="run") + child.kill() + assert child.proc.returncode==-signal.SIGKILL + + +def test_unrelated_failure_is_not_refusal(db,launch): + broken=db.rsplit('/',1)[0]+'/p026_nonexistent' + _,err=launch(broken).done(code=1) + assert not any(token in err for token in ("FENCED","LEASE-UNAVAILABLE","LEASE-EXPIRED")) + + +@pytest.mark.parametrize("stage",["before_restore","after_restore"]) +def test_restore_stage_recovery(db,launch,tmp_path,stage): + seed(db);launch(db).done() + c=connect(db) + with c.cursor() as cur:cur.execute("INSERT INTO votes(zid,pid,tid,vote,created) VALUES(1,0,0,1,2000)") + c.close() + child=launch(db,stage=stage,directory=tmp_path/'restore',extra={'P026_LEASE_SECONDS':'2'}) + ack=child.ack();child.kill() + assert lease(db)['unexpired'] # the dead owner's lease is left live, as R05 requires + repair_after_unclean_death(launch,db, + lambda: rows(db)['math_ticks']['input_checkpoint']['event_count']==25,why=f'restore repair at {stage}') + assert_coherent(db) + (ARTIFACTS/f'stage-{stage}.json').write_text(json.dumps(dict(stage=stage,ack=ack,signal='SIGKILL',restart='passed',manual_lease_expiry=False,postcondition='coherent independent fold'))) + + +@pytest.mark.parametrize("stage",["before_cursor","after_cursor","before_sweep","after_sweep"]) +def test_reader_stages(db,launch,tmp_path,stage): + seed(db);launch(db).done() + child=launch(db,'reader',stage=stage,directory=tmp_path/'reader') + ack=child.ack();child.kill() + seen=set() + for _ in range(3): + out,_=launch(db,'reader').done() + seen.update(b['payloads']['main']['zid'] for b in json.loads(out)) + assert seen=={1} + (ARTIFACTS/f'stage-{stage}.json').write_text(json.dumps(dict(stage=stage,ack=ack,signal='SIGKILL',restart='passed',postcondition='reader redelivered complete bundle'))) + + +@pytest.mark.parametrize("stage",["bundle_pinned","before_companion_join"]) +def test_pinned_bundle_survives_replacement(db,launch,tmp_path,stage): + seed(db);launch(db).done() + old=rows(db)['math_main']['math_tick'] + child=launch(db,'read',stage=stage,directory=tmp_path/'read',args=(1,)) + ack=child.ack() + c=connect(db) + with c.cursor() as cur:cur.execute("INSERT INTO votes(zid,pid,tid,vote,created) VALUES(1,0,0,1,2000)") + c.close();launch(db).done() + child.release();out,_=child.done();bundle=json.loads(out) + assert bundle['math_tick']==old + p=bundle['payloads'];assert MAPPING(p['main'],p['bidtopid'],p['ptptstats'])==[] + (ARTIFACTS/f'stage-{stage}.json').write_text(json.dumps(dict(stage=stage,ack=ack,action='atomic replacement by second process; release',postcondition='old pinned bundle remains coherent'))) + + +def test_r12_real_reader_recovers_after_advance(db,launch,tmp_path): + for zid in (1,2,3,4):seed(db,zid) + a=launch(db,stage='after_main',directory=tmp_path/'late',extra={'POLL_ALLOWLIST':'1'}) + a.ack();launch(db,extra={'POLL_ALLOWLIST':'2,3,4'}).done() + seen=set() + for _ in range(4): + out,_=launch(db,'reader').done();seen.update(b['payloads']['main']['zid'] for b in json.loads(out)) + assert seen=={2,3,4} + a.release();a.done() + for _ in range(4): + out,_=launch(db,'reader').done();seen.update(b['payloads']['main']['zid'] for b in json.loads(out)) + assert seen=={1,2,3,4} + + +def test_r07_disjoint_shards_cover_every_zid(db,launch): + for zid in range(1,7):seed(db,zid) + a=launch(db,extra={'POLL_SHARD_INDEX':'0','POLL_SHARD_COUNT':'2'}) + b=launch(db,extra={'POLL_SHARD_INDEX':'1','POLL_SHARD_COUNT':'2'}) + a.done();b.done() + for zid in range(1,7): + tables=assert_coherent(db,zid) + assert tables['math_main']['math_tick']==0 + + +def test_r07_old_new_shard_overlap_is_fenced(db,launch,tmp_path): + for zid in (2,4,6):seed(db,zid) + old=launch(db,stage='after_lease',directory=tmp_path/'old-shard',extra={'POLL_SHARD_INDEX':'0','POLL_SHARD_COUNT':'2','POLL_ALLOWLIST':'2'}) + old.ack() + new=launch(db,extra={'POLL_SHARD_INDEX':'2','POLL_SHARD_COUNT':'3','POLL_ALLOWLIST':'2'}) + # The old shard owner is still live, so the overlap is refused as unavailable. + _,err=new.done(code=4);assert 'LEASE-UNAVAILABLE' in err + old.release();old.done() + launch(db,extra={'POLL_SHARD_COUNT':'3','POLL_SHARD_INDEX':'0'}).done() + launch(db,extra={'POLL_SHARD_COUNT':'3','POLL_SHARD_INDEX':'1'}).done() + for zid in (2,4,6):assert_coherent(db,zid) + + +def test_r08_exact_duplicate_multiplicity(db,launch): + seed(db);launch(db).done() + c=connect(db) + with c.cursor() as cur: + cur.execute('SET session_replication_role=replica') # admit a legacy duplicate, bypass latest-unique maintenance + cur.execute('INSERT INTO votes(zid,pid,tid,vote,created) SELECT zid,pid,tid,vote,created FROM votes WHERE zid=1 AND pid=0 AND tid=0') + c.close();launch(db).done() + t=assert_coherent(db) + assert t['math_ticks']['input_checkpoint']['event_count']==25 + assert t['math_main']['data']['user-vote-counts']['0']==4 + + +def test_failed_zid_does_not_starve_source_sweep(db,launch): + seed(db,1);seed(db,2) + c=connect(db) + with c.cursor() as cur:cur.execute('UPDATE votes SET vote=NULL WHERE zid=1 AND pid=0 AND tid=0') + child=launch(db,'run',extra={'P026_POLL_MS':'20'}) + deadline=time.monotonic()+30 + while not rows(db,2)['math_main'] and time.monotonic()=1 + cur.execute('UPDATE votes SET vote=-1 WHERE zid=1 AND pid=0 AND tid=0') + deadline=time.monotonic()+30 + while not rows(db,1)['math_main'] and time.monotonic() 'rustproto')") + _,err=launch(db).done(code=1) + assert 'coordinator failed' in err + assert all(v is None for v in rows(db).values()) + with c.cursor() as cur:cur.execute('ALTER TABLE math_main DROP CONSTRAINT p026_reject') + assert lease(db)['unexpired'] is False + launch(db).done();assert_coherent(db);c.close() + + +def test_r08_reference_warm_cache_negative_control(db,launch): + import os,subprocess,sys + from coordinator.conftest import ROOT + seed(db) + env=dict(os.environ,DATABASE_URL=db,PYTHONPATH=str(ROOT/'delphi'),OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1') + ref=subprocess.run([sys.executable,str(ROOT/'delphi/tests/coordinator/reference_child.py'),'--warm-boundary'],env=env,capture_output=True,text=True,timeout=120) + assert ref.returncode==0,(ref.stdout,ref.stderr) + c=connect(db) + with c.cursor() as cur: + cur.execute('SELECT pid,tid,vote,created FROM votes WHERE zid=1 ORDER BY created') + events=[dict(zip(('pid','tid','vote','created'),r)) for r in cur] + c.close() + assert len(events)==26 + problems=FOLD.check_published_against_fold(rows(db,env='python')['math_main']['data'],FOLD.fold_votes(events)) + assert problems, 'warm strict-watermark reference must miss the committed boundary vote' + launch(db).done();assert_coherent(db) + + +def test_actual_worker_sigkill_is_not_success(db,launch,tmp_path): + import os + seed(db) + child=launch(db,stage='before_worker_apply',directory=tmp_path/'worker-death') + ack=child.ack();worker_pid=ack['context']['worker_pid'] + assert worker_pid!=child.proc.pid + os.kill(worker_pid,signal.SIGKILL) + child.release();child.done(code=1) + assert all(v is None for v in rows(db).values()) + assert lease(db)['unexpired'] is False # a clean failure releases its own epoch + launch(db).done();assert_coherent(db) + + +def test_terminate_actual_publication_backend_rolls_back(db,launch,tmp_path): + seed(db) + child=launch(db,stage='after_bidtopid',directory=tmp_path/'backend-death') + ack=child.ack();backend=ack['context']['backend_pid'] + c=connect(db) + with c.cursor() as cur: + cur.execute('SELECT state,xact_start,query FROM pg_stat_activity WHERE pid=%s',(backend,)) + state,start,query=cur.fetchone() + assert state=='idle in transaction' and start is not None and 'math_bidtopid' in query + cur.execute('SELECT pg_terminate_backend(%s)',(backend,));assert cur.fetchone()[0] + child.release();child.done(code=1) + assert all(v is None for v in rows(db).values()) + assert lease(db)['unexpired'] is False # a clean failure releases its own epoch + launch(db).done();assert_coherent(db);c.close() diff --git a/delphi/tests/coordinator/test_reference_controls.py b/delphi/tests/coordinator/test_reference_controls.py new file mode 100644 index 0000000000..ce1e483a3a --- /dev/null +++ b/delphi/tests/coordinator/test_reference_controls.py @@ -0,0 +1,32 @@ +"""Preserve C's original R12 SQL negative controls without changing assertions.""" +import ast +import json +import pytest +import sqlalchemy as sa +from coordinator.conftest import asset, seed + + +def controls(): + tree=ast.parse(asset('test_r12_publication_cursor.py')) + selected=[] + names={'allocate_and_write','Prefetcher','test_out_of_order_commits_of_equal_ticks_across_many_zids','TestNegativeControl'} + for n in tree.body: + if isinstance(n,(ast.FunctionDef,ast.ClassDef)) and n.name in names:selected.append(n) + elif isinstance(n,ast.Assign) and any(isinstance(t,ast.Name) and t.id=='_WRITE_MAIN_SQL' for t in n.targets):selected.append(n) + def seed_reference(engine,zid=1,n_ptpts=4,n_cmts=3): + seed(engine.url.render_as_string(hide_password=False),zid,n_ptpts,n_cmts) + scope={'sa':sa,'json':json,'MATH_ENV':'recovery','seed_conversation':seed_reference} + exec(compile(ast.Module(body=selected,type_ignores=[]),'unchanged-r12-controls.py','exec'),scope) + return scope + + +@pytest.mark.parametrize('case',['equal-ticks-many-zids','sequence-alone-sees-two']) +def test_original_r12_controls(db,case): + scope=controls() + engine=sa.create_engine(db,poolclass=sa.pool.NullPool) + try: + if case=='equal-ticks-many-zids': + scope['test_out_of_order_commits_of_equal_ticks_across_many_zids'](engine,db) + else: + scope['TestNegativeControl']().test_a_sequence_allocated_cursor_is_unique(engine,db) + finally:engine.dispose() diff --git a/delphi/tests/coordinator/test_s1_identity.py b/delphi/tests/coordinator/test_s1_identity.py new file mode 100644 index 0000000000..09ca40675e --- /dev/null +++ b/delphi/tests/coordinator/test_s1_identity.py @@ -0,0 +1,200 @@ +"""S1 original-byte custody and genuine severed COMMIT acknowledgement witnesses.""" +import hashlib +import json +import socket +import struct +import threading +from urllib.parse import urlsplit, urlunsplit + +import pytest +from coordinator.conftest import connect, rows, seed +from coordinator.test_store import fixture_file + + +@pytest.mark.parametrize("table", ["math_main", "math_bidtopid", "math_ptptstats"]) +@pytest.mark.parametrize("corrupt", ["jsonb", "original", "original_with_digest", "digest"]) +def test_original_jsonb_corruption_is_detected(db, launch, table, corrupt): + seed(db) + launch(db).done() + c = connect(db) + with c.cursor() as cur: + if corrupt == "jsonb": + cur.execute(f"UPDATE {table} SET data=data || '{{\"synthetic_corruption\":true}}'::jsonb") + elif corrupt == "digest": + cur.execute(f"UPDATE {table} SET original_sha256=%s", ("0" * 64,)) + else: + cur.execute(f"SELECT original_bytes FROM {table}") + raw = json.loads(bytes(cur.fetchone()[0])) + raw["synthetic_corruption"] = True + raw = json.dumps(raw).encode() + cur.execute(f"UPDATE {table} SET original_bytes=%s", (raw,)) + if corrupt == "original_with_digest": + # Defeat both raw-hash checks; decoded correspondence still fails. + digest = hashlib.sha256(raw).hexdigest() + cur.execute(f"UPDATE {table} SET original_sha256=%s", (digest,)) + cur.execute("UPDATE math_ticks SET input_checkpoint=jsonb_set(input_checkpoint,%s,to_jsonb(%s::text))", + (["original_digests", table.removeprefix("math_")], digest)) + c.close() + launch(db, "read", args=(1,)).done(code=1) + launch(db, extra={"P026_INCREMENTAL": "0"}).done() + launch(db, "read", args=(1,)).done() + + +def test_exact_worker_bytes_and_numeric_spelling_survive_publication(db, launch, tmp_path): + seed(db) + launch(db).done() + before = rows(db) + payloads = {key: before["math_" + key]["data"] for key in ("main", "bidtopid", "ptptstats")} + raw = {key: json.dumps(value, indent=2).encode() + b"\n" for key, value in payloads.items()} + # A lexical representation which JSONB will erase, including negative zero. + raw["main"] = raw["main"].rstrip()[:-1] + b', "synthetic_lexical": [-0.0, 1e-7, 1.000]}\n' + payloads["originals"] = {key: list(value) for key, value in raw.items()} + launch(db, "publish-fixture", args=(fixture_file(tmp_path, payloads, expected=0),)).done() + out, _ = launch(db, "read", args=(1,)).done() + bundle = json.loads(out) + current = rows(db) + for key, original in raw.items(): + row = current["math_" + key] + assert bytes(row["original_bytes"]) == original + assert bytes(bundle["payloads"]["originals"][key]) == original + assert row["original_sha256"] == hashlib.sha256(original).hexdigest() + assert json.loads(original) == row["data"] + assert bundle["publisher_epoch"] == current["math_ticks"]["publisher_epoch"] + assert bundle["operation_id"] == current["math_ticks"]["operation_id"] + + +class CommitProxy: + """Loopback PG wire relay: swallow the publication's actual COMMIT reply. + + No coordinator fault stage or Python worker patch: observe Parse/Query frames + for math_ticks, then drop CommandComplete(COMMIT) on that connection. The + backend has committed; the coordinator remains awaiting its reply until release. + Subsequent connections (including readback and heartbeat) pass through. + """ + def __init__(self, db): + parsed = urlsplit(db) + self.target = (parsed.hostname, parsed.port) + self.listener = socket.socket() + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen() + self.listener.settimeout(.2) + self.url = urlunsplit(parsed._replace(netloc=f"postgres@127.0.0.1:{self.listener.getsockname()[1]}")) + self.committed = threading.Event() + self.release = threading.Event() + self.stop = threading.Event() + self.claim = threading.Lock() + self.sockets = [] + self.errors = [] + self.thread = threading.Thread(target=self.accept, daemon=True) + self.thread.start() + + @staticmethod + def read(sock, size): + result = b"" + while len(result) < size: + part = sock.recv(size - len(result)) + if not part: + raise EOFError + result += part + return result + + def accept(self): + while not self.stop.is_set(): + try: + client, _ = self.listener.accept() + except socket.timeout: + continue + except OSError: + break + server = socket.create_connection(self.target, timeout=10) + server.settimeout(None) + self.sockets.extend([client, server]) + threading.Thread(target=self.relay, args=(client, server), daemon=True).start() + + def relay(self, client, server): + publication = threading.Event() + def upstream(): + try: + # No TLS in this synthetic trust-authenticated fixture. + size = self.read(client, 4) + server.sendall(size + self.read(client, struct.unpack("!I", size)[0] - 4)) + while True: + kind = self.read(client, 1) + size = self.read(client, 4) + body = self.read(client, struct.unpack("!I", size)[0] - 4) + if kind in (b"P", b"Q") and b"INSERT INTO math_ticks" in body: + publication.set() + server.sendall(kind + size + body) + except (EOFError, OSError): + pass + threading.Thread(target=upstream, daemon=True).start() + try: + while True: + kind = self.read(server, 1) + size = self.read(server, 4) + body = self.read(server, struct.unpack("!I", size)[0] - 4) + if (kind == b"C" and body == b"COMMIT\0" and publication.is_set() + and self.claim.acquire(blocking=False)): + self.committed.set() + if not self.release.wait(60): + self.errors.append("COMMIT release missing") + client.shutdown(socket.SHUT_RDWR) + server.shutdown(socket.SHUT_RDWR) + return + client.sendall(kind + size + body) + except (EOFError, OSError): + pass + + def close(self): + self.release.set() + self.stop.set() + self.listener.close() + for sock in self.sockets: + sock.close() + self.thread.join(2) + + +@pytest.mark.parametrize("replacement", [False, True]) +def test_uncertain_commit_readback_binds_publishing_epoch(db, launch, tmp_path, replacement): + seed(db) + proxy = CommitProxy(db) + try: + child = launch(proxy.url) + assert proxy.committed.wait(30), "publication COMMIT not intercepted" + original = rows(db) + assert original["math_ticks"]["math_tick"] == 0 + if replacement: + # Another real publisher acquires the next epoch and fills the same + # tick after loss of current rows, with identical content and op id. + # This models exactly the content-only readback hole, without using + # the production implementation to construct the expected verdict. + c = connect(db) + with c.cursor() as cur: + for table in ("math_main", "math_bidtopid", "math_ptptstats", "math_ticks"): + cur.execute(f"DELETE FROM {table} WHERE math_env='rustproto'") + cur.execute("UPDATE coordinator_leases SET expires_at=clock_timestamp()-interval '1 second'") + c.close() + payloads = {key: original["math_" + key]["data"] for key in ("main", "bidtopid", "ptptstats")} + payloads["originals"] = {key: list(bytes(original["math_" + key]["original_bytes"])) + for key in payloads} + path = fixture_file(tmp_path, payloads) + fixture = json.loads(path.read_text()) + fixture["checkpoint"] = original["math_ticks"]["input_checkpoint"] + path.write_text(json.dumps(fixture)) + launch(db, "publish-fixture", args=(path,)).done() + successor = rows(db) + assert successor["math_ticks"]["math_tick"] == 0 + assert successor["math_ticks"]["publisher_epoch"] > original["math_ticks"]["publisher_epoch"] + assert successor["math_ticks"]["operation_id"] == original["math_ticks"]["operation_id"] + assert successor["math_main"]["data"] == original["math_main"]["data"] + proxy.release.set() + out, err = child.done(code=1 if replacement else 0) + if replacement: + assert "UNCERTAIN_COMMIT_LOST" in err + assert rows(db) == successor + else: + assert json.loads(out)["published"] == 1 + assert rows(db) == original + assert not proxy.errors + finally: + proxy.close() diff --git a/delphi/tests/coordinator/test_step1_review_controls.py b/delphi/tests/coordinator/test_step1_review_controls.py new file mode 100644 index 0000000000..33958f8a3d --- /dev/null +++ b/delphi/tests/coordinator/test_step1_review_controls.py @@ -0,0 +1,181 @@ +"""Astra's step-1 review controls (board [256]), re-pinned against the fixes. + +`cost-reduction/scripts/p2727-step1-astra-review.py` characterised two defects +at 910ba8a15 and two ownership positives. The two defect assertions are inverted +here — they now assert the required behaviour — and the two positive controls +are kept as the review wrote them, so the same four seams stay covered. + +F1: a bundle resident in the warm cache must not be trusted for the existence or + generation of the durable companions (CO02/CO06, contract Rev6 "independent + resident-cache integrity reconciliation"). +F2: `publish-fixture` must report a typed ownership refusal through the same + exit mapper as `once`, not print it and exit 0. +""" +import json +import os +import sys + +import pytest +from coordinator.conftest import assert_coherent, connect, lease, rows, seed, wait + + +def query(db, sql, args=()): + c = connect(db) + try: + with c.cursor() as cur: + cur.execute(sql, args) + return cur.fetchall() if cur.description else None + finally: + c.close() + + +def test_resident_cache_does_not_hide_a_deleted_companion(db, launch): + """F1, re-pinned. The review deleted math_bidtopid under a warm cache and + watched six further passes leave it missing. The resident bundle is now + reconciled against the store's generation and companion existence on every + hit, so the daemon repairs with no new input and no restart.""" + seed(db) + child = launch(db, "run", extra={"P026_POLL_MS": "50", "P026_CACHE_CAP": "2"}) + wait(lambda: lease(db) and lease(db)["owner_epoch"] >= 4, alive=child, + why="cache warmed across passes") + before = rows(db)["math_main"]["math_tick"] + query(db, "DELETE FROM math_bidtopid WHERE math_env='rustproto' AND zid=1") + wait(lambda: rows(db)["math_bidtopid"] is not None, alive=child, + why="resident-cache integrity reconciliation repairs the deleted companion") + child.kill() + tables = assert_coherent(db) + # Repaired by republication from the full authoritative prefix, with no + # source change: the generation advances, it is not silently reinstated. + assert tables["math_main"]["math_tick"] > before + + # Control from the review, retained: a cache-disabled run of the same + # corruption also repairs, so the fix did not merely move the problem. + query(db, "DELETE FROM math_ptptstats WHERE math_env='rustproto' AND zid=1") + launch(db, extra={"P026_CACHE_CAP": "0"}).done() + assert_coherent(db) + + +def test_resident_cache_still_skips_the_store_when_the_generation_is_intact(db, launch): + """The other half of F1: integrity reconciliation must not disable the + cache. An untouched conversation stays cached and republishes nothing.""" + seed(db) + launch(db).done() + published = rows(db)["math_main"]["math_tick"] + child = launch(db, "run", extra={"P026_POLL_MS": "50", "P026_CACHE_CAP": "2"}) + wait(lambda: lease(db) and lease(db)["owner_epoch"] >= 5, alive=child, + why="several quiet passes over a cached conversation") + child.kill() + assert rows(db)["math_main"]["math_tick"] == published + + +def fixture_file(tmp_path, db): + r = rows(db) + payloads = {name: r["math_" + name]["data"] for name in ("main", "bidtopid", "ptptstats")} + p = tmp_path / "publish.json" + p.write_text(json.dumps({"zid": 1, "expected_tick": r["math_ticks"]["math_tick"], + "checkpoint": r["math_ticks"]["input_checkpoint"], + "payloads": payloads})) + return p + + +def test_publish_fixture_reports_expiry_with_the_typed_exit_code(db, launch, tmp_path): + """F2, re-pinned: `Refused(Expired)` is exit 5, exactly as `once` reports + the same seam, and every row is still unchanged.""" + seed(db) + launch(db).done() + before = rows(db) + p = fixture_file(tmp_path, db) + child = launch(db, "publish-fixture", args=(p,), stage="before_commit", + directory=tmp_path / "expiry", extra={"P026_LEASE_SECONDS": "2"}) + child.ack() + wait(lambda: not lease(db)["unexpired"], alive=child, + why="genuine DB expiry during publication") + child.release() + out, err = child.done(code=5) + assert "Refused(Expired)" in out + assert "LEASE-EXPIRED" in err + assert rows(db) == before + + # Control from the review, retained: the same seam through `once`. + query(db, "INSERT INTO votes(zid,pid,tid,vote,created) VALUES(1,0,0,1,2000)") + control = launch(db, stage="before_commit", directory=tmp_path / "normal", + extra={"P026_LEASE_SECONDS": "2"}) + control.ack() + wait(lambda: not lease(db)["unexpired"], alive=control, + why="normal once publication expires") + control.release() + _, err = control.done(code=5) + assert "LEASE-EXPIRED" in err and rows(db) == before + + +def test_publication_refuses_when_the_remaining_lease_is_below_the_commit_margin(db, launch, tmp_path): + """Rev6 CO04: the final in-transaction authorization checks the *remaining* + lease under the row lock. A lease that is still technically unexpired but + below the margin is refused and rolled back, not gambled on.""" + seed(db) + launch(db).done() + before = rows(db) + p = fixture_file(tmp_path, db) + child = launch(db, "publish-fixture", args=(p,), stage="before_commit", + directory=tmp_path / "margin", extra={"P026_LEASE_SECONDS": "10", + "P026_COMMIT_MARGIN_SECONDS": "9.99"}) + child.ack() + child.release() + out, err = child.done(code=5) + assert "Refused(Expired)" in out + # Refused while the lease was still unexpired: this is the margin, not expiry. + assert "remaining lease below the commit margin" in err + assert rows(db) == before + + +def slow_worker(tmp_path): + script = tmp_path / "slow-worker" + script.write_text(f'''#!{sys.executable} +import os,sys,time +from pathlib import Path +from polismath import engine_adapter as a +original=a.Adapter.handle +def handle(self,req): + if req.get('op')=='compute': + Path(os.environ['P026_WORKER_READY']).write_text(str(os.getpid())) + while not Path(os.environ['P026_WORKER_GO']).exists():time.sleep(.02) + return original(self,req) +a.Adapter.handle=handle +sys.argv=[sys.argv[0]]+sys.argv[3:] +raise SystemExit(a.main()) +''') + script.chmod(0o700) + return script + + +@pytest.mark.parametrize("transfer", [False, True]) +def test_renewal_during_pending_worker_call(db, launch, tmp_path, transfer): + """Both positive controls from the review, unchanged in substance: an + actual Python worker held inside a compute request, with Rust unmodified.""" + seed(db) + ready = tmp_path / "ready" + go = tmp_path / "go" + child = launch(db, extra={"P026_PYTHON": str(slow_worker(tmp_path)), + "P026_WORKER_READY": str(ready), "P026_WORKER_GO": str(go), + "P026_LEASE_SECONDS": "2"}) + wait(ready.exists, alive=child, why="actual worker waiting inside compute call") + pid = int(ready.read_text()) + if transfer: + query(db, "UPDATE coordinator_leases SET owner_id='synthetic-successor'," + "owner_epoch=owner_epoch+1,expires_at=clock_timestamp()+interval '20 seconds' " + "WHERE zid=1 AND math_env='rustproto'") + _, err = child.done(code=3) + assert "FENCED" in err and all(v is None for v in rows(db).values()) + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + else: + initial = query(db, "SELECT expires_at FROM coordinator_leases " + "WHERE zid=1 AND math_env='rustproto'")[0][0] + wait(lambda: query(db, "SELECT expires_at > %s + interval '3 seconds' FROM " + "coordinator_leases WHERE zid=1 AND math_env='rustproto'", + (initial,))[0][0], + alive=child, why="renewal extends beyond original lease window") + assert lease(db)["unexpired"] and all(v is None for v in rows(db).values()) + go.write_text("go") + child.done() + assert_coherent(db) diff --git a/delphi/tests/coordinator/test_step2_review_controls.py b/delphi/tests/coordinator/test_step2_review_controls.py new file mode 100644 index 0000000000..643a57e84d --- /dev/null +++ b/delphi/tests/coordinator/test_step2_review_controls.py @@ -0,0 +1,151 @@ +"""Astra's round-2 review controls (board [276]), re-pinned against the fixes. + +`cost-reduction/scripts/p2727-r2-astra-review.py` characterised three defects at +ef0679de0 and one actual-route control. The defect assertions are inverted here — +they now assert the required behaviour — and the route control is kept as written, +driven by the committed copy of its Node harness. + +R2-F1: persisted payload corruption in any of the three published tables must be + repaired by the full-source ceiling, not survive behind a resident bundle. +R2-F2: the recorded source age must run from before the source read, so a long + compute cannot reset it and buy another fast-path interval. +R2-F3: the pca2 route serves a committed generation of zero; only the cold + `getPca(zid, undefined)` helper call misses it. +R2-F4: the failure gauges must be scoped to this shard and allowlist, like the + source-age gauges. +""" +import json +import os +import subprocess + +import pytest +from coordinator.conftest import ROOT, assert_coherent, connect, lease, rows, seed, wait +from coordinator.test_incremental import emf, query + +ROUTE_PROBE = ROOT / "coordinator-rs/tools/node_route_probe.cjs" +EVIDENCE = ROOT / "coordinator-rs/evidence" + +requires_server_modules = pytest.mark.skipif( + not (ROOT / "server/node_modules").exists(), + reason="server/node_modules is absent; install or link it to run the real Node route", +) + + +@pytest.mark.parametrize("table", ["math_main", "math_bidtopid", "math_ptptstats"]) +def test_resident_payload_corruption_is_repaired_by_the_full_source_ceiling(db, launch, table): + """R2-F1 re-pinned. The metadata check cannot see a mutated payload, so the + authoritative path re-reads and re-hashes the persisted generation from the + store instead of trusting the resident bundle.""" + seed(db) + child = launch(db, "run", extra={"P026_POLL_MS": "100", "P026_CACHE_CAP": "2", + "P026_RECONCILE_SECONDS": "1"}) + wait(lambda: lease(db) and lease(db)["owner_epoch"] >= 3, alive=child, + why="resident Bundle warmed") + before = rows(db)["math_main"]["math_tick"] + query(db, f"UPDATE {table} SET data=data || '{{\"p026_corrupt\":true}}'::jsonb " + "WHERE zid=1 AND math_env='rustproto'") + wait(lambda: rows(db)["math_main"]["math_tick"] > before, alive=child, + why="the ceiling validates persisted payloads and republishes") + child.kill() + repaired = assert_coherent(db) + assert "p026_corrupt" not in repaired[table]["data"] + assert repaired["math_main"]["math_tick"] > before + # Control from the review, retained: a cold read of a corrupted generation + # is still rejected rather than served. + query(db, f"UPDATE {table} SET data=data || '{{\"p026_corrupt\":true}}'::jsonb " + "WHERE zid=1 AND math_env='rustproto'") + _, err = launch(db, "read", args=(1,)).done(code=1) + assert "absent/inconsistent bundle" in err + + +def test_recorded_source_age_starts_before_the_source_read(db, launch, tmp_path): + """R2-F2 re-pinned. The stored reconciled_at is the probe's database time, + which precedes the snapshot, so a compute that outlasts the ceiling cannot + advertise a fresh source age or earn another fast-path interval.""" + seed(db) + child = launch(db, stage="after_source_selection", directory=tmp_path / "snapshot", + extra={"P026_RECONCILE_SECONDS": "1"}) + child.ack() + taken = query(db, "SELECT clock_timestamp()")[0][0] + # Invisible to every aggregate in the probe, and committed after the + # snapshot this process already selected. + query(db, "UPDATE votes SET vote=1 WHERE zid=1 AND pid=0 AND tid=0") + query(db, "UPDATE votes SET vote=-1 WHERE zid=1 AND pid=0 AND tid=1") + wait(lambda: query(db, "SELECT clock_timestamp() > %s + interval '3 seconds'", (taken,))[0][0], + alive=child, why="selected snapshot older than the reconciliation ceiling") + child.release() + child.done() + # The record is stamped at or before the snapshot, never at completion. + assert query(db, "SELECT reconciled_at <= %s FROM coordinator_reconciliation " + "WHERE zid=1 AND math_env='rustproto'", (taken,))[0][0] + before = rows(db)["math_main"]["math_tick"] + out = tmp_path / "age.jsonl" + launch(db, extra={"P026_RECONCILE_SECONDS": "2", "P026_METRICS": str(out)}).done() + record = emf(out, "source_pass")[-1] + assert record["SourcePassSkipped"] == 0, "a stale snapshot must not be credited as fresh" + assert record["SourcePassReconciled"] == 1 + assert record["SourcePassPublished"] == 1 + assert rows(db)["math_main"]["math_tick"] > before + assert_coherent(db) + + +def test_failure_gauges_respect_candidate_scope(db, launch, tmp_path): + """R2-F4 re-pinned. A conversation this process would never attempt is not + this process's backlog.""" + seed(db, 1) + seed(db, 2) + query(db, "INSERT INTO coordinator_failures(math_env,zid,attempts,first_failed_at,next_attempt) " + "VALUES('rustproto',2,1,clock_timestamp()-interval '1 hour',clock_timestamp())") + output = tmp_path / "scoped.jsonl" + launch(db, extra={"POLL_ALLOWLIST": "1", "P026_METRICS": str(output)}).done() + r = emf(output, "source_pass")[-1] + assert r["SourcePassConversations"] == 1 + assert r["ReconciliationBacklogConversations"] == 0 + assert r["FailureBacklogConversations"] == 0, "zid 2 is not a candidate of this pass" + assert r["OldestUnrepairedAgeSeconds"] == 0 + # ... and it is counted by a process whose allowlist does include it. + both = tmp_path / "both.jsonl" + launch(db, extra={"POLL_ALLOWLIST": "1,2", "P026_METRICS": str(both)}).done() + r = emf(both, "source_pass")[-1] + assert r["FailureBacklogConversations"] == 1 + assert r["OldestUnrepairedAgeSeconds"] >= 3600 + + +@requires_server_modules +def test_actual_pca2_route_serves_generation_zero(db, launch, tmp_path): + """R2-F3, kept as the review wrote it. The route supplies math_tick = -1 and + serves a committed generation of zero with 200 and ETag "0"; a matching + conditional gets 304. Only the cold `getPca(zid, undefined)` call misses it, + and it stops missing it once the route has warmed the cache. The PostgreSQL + bigint reaches this driver as the string "0", which is truthy, so the column + override is not skipped.""" + seed(db) + launch(db).done() + assert rows(db)["math_main"]["math_tick"] == 0 + result = subprocess.run( + ["node", str(ROUTE_PROBE), str(ROOT)], capture_output=True, text=True, timeout=300, + env=dict(os.environ, DATABASE_URL=db, MATH_ENV="rustproto", + CACHE_MATH_RESULTS="true", NODE_ENV="production")) + assert result.returncode == 0, (result.stdout, result.stderr) + evidence = json.loads(result.stdout) + (EVIDENCE / "d4-generation-zero.json").write_text(json.dumps({ + "profile": "real handle_GET_math_pca2 on a real Express app over loopback HTTP, " + "real reader/cache/PostgreSQL; only the parameter middleware is synthetic", + "committed_math_tick": 0, + "route": evidence, + "note": "the route requests latest with math_tick=-1 and serves generation zero; " + "getPca(zid, undefined) is a different branch and is cache-dependent. " + "That helper asymmetry is a reader defect for the server owner, " + "characterised here, not fixed here.", + }, indent=2, sort_keys=True)) + assert evidence["pgTickType"] == "string" + assert evidence["whole"]["status"] == 200 + assert evidence["whole"]["etag"] == '"0"' + assert evidence["whole"]["bodyTick"] == 0 + assert evidence["whole"]["contentEncoding"] == "gzip" + assert evidence["subset"]["status"] == 200 + assert evidence["subset"]["bodyTick"] == 0 + assert evidence["conditional"]["status"] == 304 + # The helper's cold/warm asymmetry, characterised rather than endorsed. + assert evidence["coldUndefinedPresent"] is False + assert evidence["warmUndefinedPresent"] is True diff --git a/delphi/tests/coordinator/test_store.py b/delphi/tests/coordinator/test_store.py new file mode 100644 index 0000000000..aff9f2fdb7 --- /dev/null +++ b/delphi/tests/coordinator/test_store.py @@ -0,0 +1,82 @@ +import json +import os +from pathlib import Path +import subprocess +from coordinator.conftest import ROOT,seed,connect,rows,assert_coherent + + +def fixture_file(tmp_path,payloads,expected=None): + path=tmp_path/'payload.json' + path.write_text(json.dumps(dict(zid=1,expected_tick=expected,checkpoint={'fixture':'synthetic-golden'},payloads=payloads),allow_nan=False)) + return path + + +def test_rust_python_postgres_row_shape_golden(db,launch,tmp_path): + from polismath.conversation.conversation import Conversation + from polismath.engine_adapter import emit_payloads + seed(db) + conv=Conversation(1,last_updated=1) + conv=conv.update_votes({'votes':[dict(pid=p,tid=t,vote=1 if p%2 else -1,created=1000+p*4+t) for p in range(6) for t in range(4)],'lastVoteTimestamp':1023},recompute=False) + conv=conv.recompute() + payload=emit_payloads(conv,1) + path=fixture_file(tmp_path,payload) + launch(db,'publish-fixture',args=(path,)).done() + from polismath.database.postgres import PostgresClient,PostgresConfig + pg=PostgresClient(PostgresConfig(url=db,math_env='python',ssl_mode='disable')) + # Both writers receive exactly the same Python-emitted blob input. + tick=pg.increment_math_tick(1) + pg.write_math_bidtopid(1,payload['bidtopid'],math_tick=tick) + pg.write_participant_stats(1,payload['ptptstats'],math_tick=tick) + pg.write_math_main(1,payload['main'],last_vote_timestamp=payload['main']['lastVoteTimestamp'],math_tick=tick) + pg.shutdown() + c=connect(db) + with c.cursor() as cur: + for table in ('math_main','math_bidtopid','math_ptptstats'): + cur.execute(f"SELECT data::text,math_tick FROM {table} WHERE zid=1 ORDER BY math_env") + a,b=cur.fetchall() + assert a==b,table # identical PostgreSQL JSONB text bytes, not just dicts + c.close() + + +def test_expected_tick_conflict_preserves_current(db,launch,tmp_path): + seed(db);launch(db).done() + before=rows(db) + p={k:before[t]['data'] for k,t in [('main','math_main'),('bidtopid','math_bidtopid'),('ptptstats','math_ptptstats')]} + out,_=launch(db,'publish-fixture',args=(fixture_file(tmp_path,p,expected=999),)).done() + assert 'Conflict' in out + assert rows(db)==before + + +def test_release_binary_rejects_fault_descriptor_before_db(): + binary=ROOT/'coordinator-rs/target/release/polis-coordinator' + env=dict(os.environ,P026_FAULT_DIR='/nonexistent/synthetic-control') + env.pop('DATABASE_URL',None) + r=subprocess.run([str(binary),'once'],env=env,capture_output=True,text=True) + assert r.returncode==1 and 'FAULT-CONTROL-REFUSED' in r.stderr + assert 'database' not in r.stderr.lower() + + +def test_coherent_point_read_and_quiet_noop(db,launch): + seed(db);launch(db).done() + before=rows(db) + out,_=launch(db,'read',args=(1,)).done() + assert json.loads(out)['math_tick']==before['math_main']['math_tick'] + launch(db).done() + assert rows(db)==before + + +def test_release_build_cannot_enable_fault_feature(): + env=dict(os.environ,CARGO_HOME='/private/tmp/p026-toolchain/cargo',RUSTUP_HOME='/private/tmp/p026-toolchain/rustup') + r=subprocess.run(['/private/tmp/p026-toolchain/cargo/bin/cargo','check','--manifest-path',str(ROOT/'coordinator-rs/Cargo.toml'),'--release','--features','fault-injection'],env=env,capture_output=True,text=True,timeout=90) + assert r.returncode!=0 and 'fault-injection is forbidden in release builds' in r.stderr + + +def test_missing_checkpoint_metadata_is_repaired(db,launch): + seed(db);launch(db).done() + c=connect(db) + with c.cursor() as cur: + cur.execute("UPDATE math_ticks SET input_checkpoint=NULL WHERE zid=1 AND math_env='rustproto'") + c.close() + launch(db,'read',args=(1,)).done(code=1) + launch(db).done() + assert_coherent(db) diff --git a/delphi/tests/replay_harness/test_certify_rust_admission_sweep.py b/delphi/tests/replay_harness/test_certify_rust_admission_sweep.py new file mode 100644 index 0000000000..365f345411 --- /dev/null +++ b/delphi/tests/replay_harness/test_certify_rust_admission_sweep.py @@ -0,0 +1,131 @@ +"""P-045 round 8: property-style admission sweep. + +Feeds randomly-generated malformed JSON shapes into EVERY public bridge entry +point that returns a failure list, and asserts no exception escapes — the whole +value is graded. hypothesis is not installed in this venv, so this uses a seeded +recursive generator (deterministic). It also mutates valid factory evidence with +random values (including out-of-f64-range integers that exercise the digest +refusal path) so the sweep reaches the deep custody/digest code, not only the +outer guards. +""" + +from __future__ import annotations + +import importlib.util +import random +from pathlib import Path + +from polismath.replay import coordinator_driver as cd + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(name + ".py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +_manifest = _load("test_certify_rust_identity")._manifest +_bundle = _load("test_certify_rust_publication")._bundle + +# Strings that are NOT UTF-8-encodable (lone surrogates) or are valid non-BMP, +# used in every string position (keys and values). Lone surrogates must be graded +# (serde cannot hold them); valid astral scalars must encode/hash. +_SURROGATES = ["\ud800", "\udfff", "\udc00", "a\ud800b", "🐀pair", + chr(0x1F600), "astral \U0001d538 key"] +_ATOMS = [ + None, True, False, 0, 1, -1, 7, + 2 ** 63, 2 ** 64, 2 ** 64 + 1, -(2 ** 63) - 1, 10 ** 25, 10 ** 400, -(10 ** 400), + 0.0, -0.0, 1.5, 1e-7, 1e21, float("inf"), float("nan"), + "", "x", "é", "a\nb\t\"c\\", "\x00\x1f", "sha", "a" * 64, + [], {}, [1], {"x": 1}, {"slot": 0}, {"sha256": []}, +] + _SURROGATES + +_KEYS = ["a", "b", "x", "slot", "sha256", "data", "id", "members", "count", + "bidToPid", "zid", "schema", "cursors", "votes", "operation_id", + "publisher_epoch", "original_digests", "payload_digests", + "base-clusters", "group-clusters"] + _SURROGATES + + +def _rand_json(rng, depth=0): + if depth >= 4 or rng.random() < 0.45: + return rng.choice(_ATOMS) + if rng.random() < 0.5: + return [_rand_json(rng, depth + 1) for _ in range(rng.randint(0, 4))] + return {rng.choice(_KEYS): _rand_json(rng, depth + 1) for _ in range(rng.randint(0, 4))} + + +def _graded(result): + return isinstance(result, list) and all(isinstance(f, str) for f in result) + + +def _call_all(value, rng): + """Every public failure-list entry point, on `value` as the outer container + and (for readback/observer) as a substituted row inside a valid skeleton.""" + calls = 0 + assert _graded(cd.validate_s1_identity(value)); calls += 1 + assert _graded(cd.validate_s1_identity( + value, expected_admission=value if isinstance(value, dict) else {}, + expected_identity=value if isinstance(value, dict) else {}, + expected_cursors=value, expected_files=value)); calls += 1 + assert _graded(cd.validate_readback( + value, expected_prior_tick=rng.choice([None, 0, 3, True]), + operation_id=rng.choice(["op-1", 7, None]), + publisher_epoch=rng.choice([5, True, "x"]), + expected_input_checkpoint=value if isinstance(value, dict) else None)); calls += 1 + assert _graded(cd.observe_bundle_coherence(value)); calls += 1 + # substitute the random value into each slot of a valid bundle/manifest + for slot in ("main", "bidtopid", "ptptstats", "ticks", "zid", "math_env"): + b = _bundle() + b[slot] = value + assert _graded(cd.validate_readback(b, expected_prior_tick=None, + operation_id="op-1", publisher_epoch=5)); calls += 1 + assert _graded(cd.observe_bundle_coherence(b)); calls += 1 + # mutate the store checkpoint and a payload's data (reaches digest/custody) + b = _bundle() + b["ticks"]["input_checkpoint"] = value + assert _graded(cd.validate_readback(b, expected_prior_tick=None, + operation_id="op-1", publisher_epoch=5)); calls += 1 + b = _bundle() + b["main"]["data"] = value + assert _graded(cd.validate_readback(b, expected_prior_tick=None, + operation_id="op-1", publisher_epoch=5)); calls += 1 + m = _manifest() + for slot in ("files", "math_input_cursors", "observed_state_cursors", "admission"): + mm = dict(m) + mm[slot] = value + assert _graded(cd.validate_s1_identity(mm)); calls += 1 + # a string value carried in the digested payload and the original bytes, + # plus a string custody field (reaches the serde encoder and str encode) + if isinstance(value, str): + b = _bundle() + b["main"]["data"] = {"k": value} + b["main"]["original_bytes"] = value + assert _graded(cd.validate_readback(b, expected_prior_tick=None, + operation_id="op-1", publisher_epoch=5)); calls += 1 + b = _bundle() + b["main"]["data"] = {value: 1} # the string as a payload KEY + assert _graded(cd.validate_readback(b, expected_prior_tick=None, + operation_id="op-1", publisher_epoch=5)); calls += 1 + for field in ("operation_id", "schema"): + b = _bundle() + b["ticks"]["input_checkpoint"][field] = value + assert _graded(cd.validate_readback(b, expected_prior_tick=None, + operation_id="op-1", publisher_epoch=5)); calls += 1 + return calls + + +def test_admission_sweep_no_exception_escapes(): + rng = random.Random(459) + total = 0 + for _ in range(1500): + total += _call_all(_rand_json(rng), rng) + # A fixed corpus of adversarial atoms as the outer value too, and every + # surrogate/valid-astral string injected into every string position. + for atom in _ATOMS: + total += _call_all(atom, rng) + for s in _SURROGATES: + total += _call_all(s, rng) + assert total > 20000 + # expose the count for the notes + print(f"\nADMISSION SWEEP: {total} public-entry calls, no exception escaped") diff --git a/delphi/tests/replay_harness/test_certify_rust_cli.py b/delphi/tests/replay_harness/test_certify_rust_cli.py new file mode 100644 index 0000000000..8f37575f77 --- /dev/null +++ b/delphi/tests/replay_harness/test_certify_rust_cli.py @@ -0,0 +1,403 @@ +"""P-045 slice 1 controls: the third producer is ADDITIVE, and the harness-only +bridge file is excluded from the engine tree hash. + +These need neither Postgres nor the Rust binary. The full-battery legacy +re-record byte-identity control (brief §rev2 :531) additionally needs the JVM + +uv and is gated on RUN_CLJ_INTEGRATION=1. +""" + +from __future__ import annotations + +import os + +import pytest + +from polismath.replay import certify as cert +from polismath.replay import coordinator_driver as cd + + +# --------------------------------------------------------------------------- +# The exclude list already exists; the bridge is its fifth harness sibling. +# --------------------------------------------------------------------------- +def _write(root, rel, text): + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + return p + + +def test_bridge_file_is_excluded_from_engine_tree_hash(tmp_path): + """Editing polismath/replay/coordinator_driver.py must NOT change + engine_tree_hash — so it never re-keys the Python recording cache nor + changes run_provenance.engine_tree_sha256 for a legacy two-driver command. + A genuine engine file (not on the exclude list) still moves the hash, which + proves the exclusion is real and scoped, not a blanket weakening.""" + assert "replay/coordinator_driver.py" in cert._ENGINE_TREE_EXCLUDE + _write(tmp_path, "replay/certify.py", "# harness\n") + driver = _write(tmp_path, "replay/coordinator_driver.py", "# bridge v1\n") + engine = _write(tmp_path, "conversation/conversation.py", "X = 1\n") + + before = cert.engine_tree_hash(polismath_root=tmp_path) + driver.write_text("# bridge v2 — a real edit to the bridge\nY = 2\n") + after_driver_edit = cert.engine_tree_hash(polismath_root=tmp_path) + assert after_driver_edit == before, "bridge edit must not move the engine hash" + + engine.write_text("X = 2\n") + after_engine_edit = cert.engine_tree_hash(polismath_root=tmp_path) + assert after_engine_edit != before, "a real engine edit MUST move the hash" + + +def test_certify_py_recording_key_ignores_the_bridge_file(tmp_path): + """The Python recording cache key is engine_tree_sha256 (ensure_py_recording). + Because the bridge is excluded, adding/editing it leaves that key identical — + the additivity the third producer relies on, asserted on the key itself.""" + _write(tmp_path, "replay/certify.py", "# harness\n") + _write(tmp_path, "conversation/conversation.py", "X = 1\n") + key_without_bridge = cert.engine_tree_hash(polismath_root=tmp_path) + _write(tmp_path, "replay/coordinator_driver.py", "# bridge\nBIG = 'x' * 10000\n") + key_with_bridge = cert.engine_tree_hash(polismath_root=tmp_path) + assert key_with_bridge == key_without_bridge + + +def test_provenance_engine_tree_is_bridge_invariant(): + """run_provenance.engine_tree_sha256 is what a legacy two-driver manifest + attests. It is computed over the real tree WITH the bridge present; this + pins that the value equals engine_tree_hash of the real tree, i.e. the + bridge contributed nothing to it.""" + prov = cert.run_provenance(root=cert.st.replays_root()) + assert prov["engine_tree_sha256"] == cert.engine_tree_hash() + + +# --------------------------------------------------------------------------- +# Registry: additive, and never Rust-by-relabelling-py. +# --------------------------------------------------------------------------- +def test_registry_has_exactly_three_producers(): + assert tuple(cd.REGISTRY) == cd.DRIVER_IDS == ("clj", "py", "rust") + assert cd.LEGACY_DRIVER_IDS == ("clj", "py") + + +def test_rust_is_not_py_relabelled(): + """The Rust producer must not borrow py's loader or its record function.""" + assert cd.RUST.loader == "rust" != cd.PY.loader + assert cd.RUST.record is not cd.PY.record + # And its record touches no py recording: it returns UNSUPPORTED_PROFILE. + r = cd.RUST.record({"profile": cd.PROFILE_SNAPSHOT_REBUILD}, {"checkpoints": [1, 2]}, tmp_out()) + assert r.driver == r.loader == "rust" + assert r.status is cd.ProducerStatus.UNSUPPORTED_PROFILE + assert r.observed_cuts == 0 and r.required_cuts == 2 + + +def tmp_out(): + import tempfile + from pathlib import Path + return Path(tempfile.mkdtemp()) + + +@pytest.mark.parametrize("bad", ["clj,py,py", "rust", "clj,,py", "", "clj,foo"]) +def test_parse_drivers_rejects_bad_lists(bad): + with pytest.raises(cd.BridgeError): + cd.parse_drivers(bad) + + +def test_parse_drivers_default_and_full(): + assert cd.parse_drivers("clj,py") == ("clj", "py") + assert cd.parse_drivers("clj,py,rust") == ("clj", "py", "rust") + + +# --------------------------------------------------------------------------- +# Policy (p045-policy/1): closed, never derived from errors. +# --------------------------------------------------------------------------- +def _good_policy(profile="snapshot-rebuild/1"): + return { + "schema": cd.POLICY_SCHEMA, "profile": profile, + "field_policy_sha256": "a" * 64, "approvals": ["APPROVAL-1"], + "replacement_assertions": ["REPL-1"], "required_controls": ["N01", "N08"], + "clock": {"mode": "pinned", "epoch_ms": 0}, + } + + +def test_load_policy_accepts_and_hashes_but_is_not_admitting(tmp_path): + import json + p = tmp_path / "policy.json" + p.write_text(json.dumps(_good_policy())) + pol = cd.load_policy(p) + assert pol.schema == "p045-policy/1" and pol.profile == "snapshot-rebuild/1" + assert len(pol.raw_sha256) == 64 + assert pol.required_controls == ("N01", "N08") + # The policy BODY is undefined (slice 3): shape-valid is never admission. + assert pol.admitting is False + + +@pytest.mark.parametrize("mutate", [ + lambda o: o.update(schema="p044-policy/1"), + lambda o: o.update(schema="p045-policy/2"), + lambda o: o.update(profile="battery-chain/2"), + lambda o: o.pop("field_policy_sha256"), + lambda o: o.pop("required_controls"), + lambda o: o.update(approvals="not-a-list"), + lambda o: o.update(clock=[]), + lambda o: o.update(field_policy_sha256="short"), + lambda o: o.update(field_policy_sha256="z" * 64), # correction 3: non-hex + lambda o: o.update(surprise=True), # correction 3: unknown key +]) +def test_load_policy_rejects_bad(tmp_path, mutate): + import json + o = _good_policy() + mutate(o) + p = tmp_path / "policy.json" + p.write_text(json.dumps(o)) + with pytest.raises(cd.BridgeError): + cd.load_policy(p) + + +# --------------------------------------------------------------------------- +# Bridge cache key: separate namespace, binds the documented fields. +# --------------------------------------------------------------------------- +def _cache_inputs(**over): + base = dict( + driver_id="rust", loader_id="rust", binary_tree_sha256="bin", + worker_tree_sha256="wrk", votes_sha256="v", comments_sha256=None, + moderation_sha256=None, convention="delphi", schedule_sha256="sch", + profile="snapshot-rebuild/1", initialization_schema="ones", + restore_schema="rebuild-prefix/1", migration_sha256="mig", + policy_sha256="pol", comparer_cfg_sha256="cmp", + serialization_profile="pg-json/1", + ) + base.update(over) + return base + + +def test_bridge_cache_key_is_deterministic_and_scoped(): + k1 = cd.bridge_cache_key(_cache_inputs()) + k2 = cd.bridge_cache_key(_cache_inputs()) + assert k1["key"] == k2["key"] + assert k1["schema"] == cd.BRIDGE_CACHE_SCHEMA + assert k1["bound"]["cache_schema"] == cd.BRIDGE_CACHE_SCHEMA + + +@pytest.mark.parametrize("field", [ + "driver_id", "loader_id", "binary_tree_sha256", "worker_tree_sha256", + "votes_sha256", "convention", "schedule_sha256", "profile", + "initialization_schema", "restore_schema", "migration_sha256", + "policy_sha256", "comparer_cfg_sha256", "serialization_profile", +]) +def test_bridge_cache_key_changes_when_any_bound_field_changes(field): + base = cd.bridge_cache_key(_cache_inputs())["key"] + changed = cd.bridge_cache_key(_cache_inputs(**{field: "MUTATED"}))["key"] + assert changed != base, f"cache key must bind {field}" + + +# --------------------------------------------------------------------------- +# Inventory additivity: the clj/py rows are untouched; rust is appended. +# --------------------------------------------------------------------------- +def _expected_entry(checkpoints): + from pathlib import Path + from types import SimpleNamespace + entry = SimpleNamespace(dataset="vw", schedule_id="uniform8-clojure-legacy", role=None) + spec = SimpleNamespace(coverage="full-stream") + return cert.ExpectedEntry( + entry=entry, spec=spec, votes_csv=Path("v.csv"), votes_sha="abc", + comments_csv=None, comments_sha=None, stream_end=4683, checkpoints=checkpoints, + ) + + +def test_legacy_inventory_is_unchanged_and_rust_is_additive(): + cps = [{"index": 0, "cut_slot": 585}, {"index": 1, "cut_slot": 1171}] + exp = _expected_entry(cps) + legacy = exp.inventory() + assert [r["engine"] for r in legacy] == ["clj", "py"], "legacy inventory() must stay 2-row clj/py" + + three = cd.three_producer_inventory(exp, cd.PROFILE_SNAPSHOT_REBUILD) + assert [r["engine"] for r in three] == ["clj", "py", "rust"] + # Each legacy row's own fields survive verbatim inside the extended row. + for legacy_row, three_row in zip(legacy, three[:2]): + for k, v in legacy_row.items(): + assert three_row[k] == v, f"additive row changed legacy field {k}" + # The rust row exists and is a visible non-pass (rust advertises no profile). + rust_row = three[2] + assert rust_row["engine"] == "rust" and rust_row["advertised"] is False + assert rust_row["status"] == cd.ProducerStatus.UNSUPPORTED_PROFILE.value + + +def test_three_producer_inventory_rejects_unknown_profile(): + with pytest.raises(cd.BridgeError): + cd.three_producer_inventory(_expected_entry([{"index": 0, "cut_slot": 0}]), "battery-chain/2") + + +# --------------------------------------------------------------------------- +# Correction 1: the reachable slice-1 bridge runner + report, end to end. +# --------------------------------------------------------------------------- +def _single_cut_entries(): + entries = [e for e in cert.load_battery() + if e.dataset == "vw" and e.schedule_id == "single-cut-clojure-legacy"] + assert entries, "vw single-cut entry must be in the starter battery" + return entries + + +def test_bridge_runner_emits_terminal_v2_report_with_rust_rows(tmp_path): + """The rust producer is invocable end-to-end and emits per-entry rows into a + polis-certification-run/2 report — rust UNSUPPORTED_PROFILE, references + INCONCLUSIVE, verdict INCONCLUSIVE, terminal manifest written. No engine, no + DB, no fabricated PASS.""" + report = cd.run_bridge_battery( + _single_cut_entries(), root=tmp_path, profile=cd.PROFILE_SNAPSHOT_REBUILD, + drivers=("clj", "py", "rust"), battery_path=str(cert.DEFAULT_BATTERY_PATH)) + assert report["schema"] == cd.CERTIFICATION_RUN_SCHEMA_V2 + assert report["terminal"] is True + engines = [(r["engine"], r["status"]) for r in report["producers"]] + assert ("rust", cd.ProducerStatus.UNSUPPORTED_PROFILE.value) in engines + assert ("clj", cd.ProducerStatus.INCONCLUSIVE.value) in engines + assert ("py", cd.ProducerStatus.INCONCLUSIVE.value) in engines + # three pairs per entry, none passing + assert len(report["pairs"]) == 3 + assert all(p["status"] != cd.ProducerStatus.PASS.value for p in report["pairs"]) + assert report["verdict"] == cd.ProducerStatus.INCONCLUSIVE.value + assert cd.bridge_exit_code(report) == 2 + assert (tmp_path / "certify_report_bridge.json").exists() + + +def test_bridge_report_receipt_profile_agrees_with_rows(tmp_path): + """Round 3 correction 4: reference receipts must carry the selected profile, + not an empty string beneath a row that says snapshot-rebuild/1.""" + report = cd.run_bridge_battery(_single_cut_entries(), root=tmp_path, + profile=cd.PROFILE_SNAPSHOT_REBUILD) + for r in report["producers"]: + if r["engine"] != "rust": + assert r["receipt"]["profile"] == cd.PROFILE_SNAPSHOT_REBUILD + assert report["drivers"] == ["clj", "py", "rust"] + + +def test_bridge_runner_honors_a_driver_subset(tmp_path): + """Round 3 correction 4: a py-only selection reports and iterates ONLY py — + it does not silently invoke/emit clj and rust.""" + report = cd.run_bridge_battery(_single_cut_entries(), root=tmp_path, + profile=cd.PROFILE_SNAPSHOT_REBUILD, drivers=("py",)) + assert report["drivers"] == ["py"] + assert {r["engine"] for r in report["producers"]} == {"py"} + assert report["pairs"] == [] # a single producer has no pairs + + +def test_bridge_runner_requires_both_references_for_rust(tmp_path): + with pytest.raises(cd.BridgeError): + cd.run_bridge_battery(_single_cut_entries(), root=tmp_path, + profile=cd.PROFILE_SNAPSHOT_REBUILD, drivers="clj,rust") + + +def test_bridge_runner_rejects_unknown_profile(tmp_path): + with pytest.raises(cd.BridgeError): + cd.run_bridge_battery(_single_cut_entries(), root=tmp_path, + profile="battery-chain/2", drivers="clj,py,rust") + + +import importlib.util as _ilu + +_HAVE_CLICK = _ilu.find_spec("click") is not None + + +@pytest.mark.skipif(not _HAVE_CLICK, reason="click not installed in this interpreter") +def test_cli_run_drivers_dispatches_to_bridge(tmp_path): + """`certify.py run --drivers clj,py,rust --profile ...` reaches the bridge and + exits 2 (INCONCLUSIVE) with a rust UNSUPPORTED_PROFILE row printed.""" + from click.testing import CliRunner + script = _cli_module() + runner = CliRunner() + result = runner.invoke(script.cli, [ + "run", "--drivers", "clj,py,rust", "--profile", "snapshot-rebuild/1", + "--only", "vw:single-cut-clojure-legacy", "--root", str(tmp_path)]) + assert result.exit_code == 2, result.output + assert "rust -> UNSUPPORTED_PROFILE" in result.output + assert "polis-certification-run/2" in result.output + + +@pytest.mark.skipif(not _HAVE_CLICK, reason="click not installed in this interpreter") +def test_cli_legacy_default_does_not_route_to_bridge(): + """Default `--drivers clj,py` with no profile must NOT print a bridge line — + the legacy path is unchanged.""" + from click.testing import CliRunner + script = _cli_module() + runner = CliRunner() + # --only with a nonexistent selection keeps this fast and engine-free; the + # point is only that no bridge report schema appears. + result = runner.invoke(script.cli, ["run", "--only", "nonesuch:nope"]) + assert "polis-certification-run/2" not in result.output + + +def _cli_module(): + import importlib.util + from pathlib import Path + path = Path(cert.__file__).resolve().parents[2] / "scripts" / "certify.py" + spec = importlib.util.spec_from_file_location("p045_certify_cli", str(path)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +# --------------------------------------------------------------------------- +# Correction 2 (round 3): additivity over the REAL retained clj<->py recordings. +# --------------------------------------------------------------------------- +def _find_real_recording(): + """Return (clj_dir, py_dir) for a real on-disk recording pair under the + standard recording root, or None. These are the gitignored artifacts the G12 + measurement used (delphi/real_data/.local/replays///{clj,py}); + they exist only after an engine run has populated them.""" + root = cert.st.replays_root() + if not root.exists(): + return None + for clj in sorted(root.glob("*/*/clj")): + py = clj.parent / "py" + if (clj / "step-000.blob.json").exists() and (py / "step-000.json").exists(): + return clj, py + return None + + +def _compare_behavior(clj, py, cache_root): + """The legacy comparison's OBSERVABLE behavior over a recording: either its + canonical per-step verdict payload, or its deterministic CertifyError (a real + recording can trip an existing raw gate, e.g. an alias collision) — both are + stable functions of the bytes, which is what additivity asserts.""" + import json + try: + res = cert.compare_recording_pair(clj, py, cache_root=cache_root) + return "ok", json.dumps(res["per_step"], sort_keys=True).encode() + except cert.CertifyError as exc: + return "err", f"{exc.stage}: {exc}".encode() + + +def test_additivity_over_real_retained_recordings(tmp_path): + """Round 3 correction 2: run the UNCHANGED legacy comparison over the REAL + retained clj<->py recordings on disk (no fresh re-record, so no JVM), and + assert its behavior + the raw blob digests are deterministic and the engine + hash is bridge-invariant — the excluded bridge file perturbs none of them, + on real bytes. Skips (naming the path) only when no real recording is present, + which is legitimate since the recordings are gitignored engine artifacts.""" + import hashlib + found = _find_real_recording() + if found is None: + pytest.skip(f"no real recording under {cert.st.replays_root()} " + "(generate one with certify --refresh-clj --refresh-py)") + clj, py = found + kind1, payload1 = _compare_behavior(clj, py, tmp_path / "c1") + kind2, payload2 = _compare_behavior(clj, py, tmp_path / "c2") + assert (kind1, payload1) == (kind2, payload2), "legacy comparison must be deterministic on real bytes" + # raw blob digests are a pure function of the bytes + d_clj = hashlib.sha256((clj / "step-000.blob.json").read_bytes()).hexdigest() + d_py = hashlib.sha256((py / "step-000.json").read_bytes()).hexdigest() + assert len(d_clj) == 64 and len(d_py) == 64 + # engine hash unchanged by the (excluded) bridge + assert cert.run_provenance(cert.st.replays_root())["engine_tree_sha256"] == cert.engine_tree_hash() + + root = os.environ.get("P045_REFERENCE_RECORDING_ROOT") + if root: + from pathlib import Path + pinned = Path(root) + assert _compare_behavior(pinned / "clj", pinned / "py", tmp_path / "cr") == \ + _compare_behavior(pinned / "clj", pinned / "py", tmp_path / "cr2") + + +@pytest.mark.skipif(os.environ.get("RUN_CLJ_INTEGRATION") != "1", + reason="needs the JVM + uv to FRESH re-record the two-driver battery") +def test_legacy_two_driver_battery_fresh_rerecord_byte_identical(): + # The fresh-re-record variant genuinely needs the engines; it stays gated on + # RUN_CLJ_INTEGRATION. A pinned pre-change full-battery verdict remains the + # one operator-supplied artifact for a complete byte-identity proof. + pytest.skip("fresh re-record needs the JVM + uv; see P-045 implementation notes") diff --git a/delphi/tests/replay_harness/test_certify_rust_digest_parity.py b/delphi/tests/replay_harness/test_certify_rust_digest_parity.py new file mode 100644 index 0000000000..be2f64bed9 --- /dev/null +++ b/delphi/tests/replay_harness/test_certify_rust_digest_parity.py @@ -0,0 +1,303 @@ +"""P-045 round 6: the bridge's canonical payload digest is a byte-for-byte port +of the ACTUAL Rust ``storage_digest`` (coordinator-rs/src/store.rs). + +This builds a tiny STANDALONE Rust probe from the UNCHANGED digest source +extracted from store.rs at test time (so it can never drift from the production +function, and does not require building the whole coordinator crate), then +compares the Rust output to :func:`coordinator_driver._canonical_payload_digest` +on Astra's probe inputs and on the real retained ``vw/single-cut`` blobs. It +skips (naming the reason) only when cargo/the toolchain or store.rs is absent — +the same discipline as the JVM-gated tests. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +from polismath.replay import certify as cert +from polismath.replay import coordinator_driver as cd + +_MAIN_RS = """\ +mod digest; +use anyhow::Result; +use serde_json::Value; +use std::io::{self, BufRead, Write}; +fn main() -> Result<()> { + let stdin = io::stdin(); + let stdout = io::stdout(); + let mut out = stdout.lock(); + for line in stdin.lock().lines() { + let line = line?; + if line.trim().is_empty() { continue; } + let value: Value = serde_json::from_str(&line)?; + writeln!(out, "{}", digest::storage_digest(&value)?)?; + } + Ok(()) +} +""" + +_DIGEST_HEADER = "use anyhow::Result;\nuse serde_json::Value;\nuse sha2::{Digest, Sha256};\n\n" + + +def _locked_versions(store): + """Pin the probe's serde_json/anyhow/sha2 to the coordinator's EXACT lock + entries, so the probe's serde encoder (and its transitive zmij float + formatter) cannot drift from the production digest. Returns None if the lock + is absent (falls back to caret ranges).""" + lock = store.parents[1] / "Cargo.lock" + if not lock.exists(): + return None + want = {} + name = None + for line in lock.read_text().splitlines(): + line = line.strip() + if line.startswith('name = "'): + name = line.split('"')[1] + elif line.startswith('version = "') and name in ("serde_json", "anyhow"): + want[name] = line.split('"')[1] + elif line.startswith('version = "') and name == "sha2" and "sha2" not in want: + # store.rs uses sha2 0.10.x; take the first (0.10) entry. + v = line.split('"')[1] + if v.startswith("0.10"): + want["sha2"] = v + return want if {"serde_json", "anyhow", "sha2"} <= set(want) else None + + +def _cargo_toml(store): + pinned = _locked_versions(store) + if pinned: + deps = (f'anyhow = "={pinned["anyhow"]}"\n' + f'sha2 = "={pinned["sha2"]}"\n' + f'serde_json = {{ version = "={pinned["serde_json"]}", ' + f'features = ["float_roundtrip"] }}\n') + else: + deps = ('anyhow = "1"\nsha2 = "0.10"\n' + 'serde_json = { version = "1", features = ["float_roundtrip"] }\n') + return ('[package]\nname = "p045-digest-probe"\nversion = "0.0.0"\n' + 'edition = "2024"\n[dependencies]\n' + deps) + + +def _find_cargo(): + for cand in (os.environ.get("P045_CARGO"), + "/private/tmp/p026-toolchain/cargo/bin/cargo", + shutil.which("cargo")): + if cand and Path(cand).exists(): + return cand + return None + + +def _find_store_rs(): + for parent in Path(cd.__file__).resolve().parents: + s = parent / "coordinator-rs" / "src" / "store.rs" + if s.exists(): + return s + return None + + +def _cargo_env(): + env = dict(os.environ) + home = Path("/private/tmp/p026-toolchain/cargo") + rustup = Path("/private/tmp/p026-toolchain/rustup") + if "CARGO_HOME" not in env and home.exists(): + env["CARGO_HOME"] = str(home) + if "RUSTUP_HOME" not in env and rustup.exists(): + env["RUSTUP_HOME"] = str(rustup) + return env + + +@pytest.fixture(scope="module") +def rust_probe(tmp_path_factory): + """Build the standalone Rust digest probe. + + SKIP only when a prerequisite is genuinely ABSENT (cargo not installed, or + store.rs missing). Once cargo and store.rs are present, a source/marker + incompatibility, a build failure, or a missing output binary is a test + FAILURE — a broken digest oracle must not masquerade as an unavailable + toolchain.""" + cargo = _find_cargo() + store = _find_store_rs() + if cargo is None: + pytest.skip("cargo not found (set P045_CARGO or install the toolchain)") + if store is None: + pytest.skip("coordinator-rs/src/store.rs not found") + src = store.read_text() + start = src.find("pub fn digest(") + end = src.find("/// Exact worker output") + if start == -1 or end == -1 or end <= start: + pytest.fail("store.rs digest source markers not found — the digest oracle " + "changed shape; the parity test must be updated, not skipped") + d = tmp_path_factory.mktemp("digest-probe") + (d / "src").mkdir() + (d / "src" / "digest.rs").write_text(_DIGEST_HEADER + src[start:end]) + (d / "src" / "main.rs").write_text(_MAIN_RS) + (d / "Cargo.toml").write_text(_cargo_toml(store)) + build = subprocess.run( + [cargo, "build", "--offline", "--quiet", "--manifest-path", str(d / "Cargo.toml")], + env=_cargo_env(), capture_output=True, text=True) + if build.returncode != 0: + pytest.fail(f"cargo is present but the digest probe failed to build " + f"(exit {build.returncode}); this is a digest-oracle regression, " + f"not a missing toolchain:\n{build.stderr.strip()[-600:]}") + binary = d / "target" / "debug" / "p045-digest-probe" + if not binary.exists(): + pytest.fail("cargo reported success but produced no probe binary") + return binary + + +def _rust_digests(binary, values): + inp = "".join(json.dumps(v) + "\n" for v in values) + return subprocess.check_output([str(binary)], input=inp, text=True).splitlines() + + +# Astra's exact reproduction inputs plus structural and integer-boundary ones. +_PARITY_CASES = [ + {"x": 1}, {"x": 1.0}, {"x": -0.0}, {"x": 1e-7}, {"x": "é"}, + 1, 1.0, -0.0, 1e-7, "é", True, False, None, + [1, 2.0, -0.0], {"a": 1, "b": {"c": 1e-7, "d": "z"}}, + {"z": 1, "a": 2}, {"nl": "a\nb\t\"c\\/", "unicode": "π✓"}, + {"big": 1e20, "small": 1e-20, "neg": -12.34}, {"e21": 1e21}, + # serde_json integer-token boundaries: i64/u64 exact, else f64. + {"x": 2 ** 53 + 1}, {"x": 2 ** 63}, {"x": 2 ** 63 - 1}, + {"x": 2 ** 64 - 1}, # u64::MAX (exact) + {"x": -(2 ** 63)}, # i64::MIN (exact) + {"x": 2 ** 64}, # u64::MAX + 1 -> f64 + {"x": 2 ** 64 + 1}, {"x": -(2 ** 63) - 1}, {"x": 10 ** 25 + 1}, + 2 ** 64, -(2 ** 63) - 1, +] + + +@pytest.mark.parametrize("i", range(len(_PARITY_CASES))) +def test_digest_parity_on_probe_inputs(rust_probe, i): + value = _PARITY_CASES[i] + (rust,) = _rust_digests(rust_probe, [value]) + assert cd._canonical_payload_digest(value) == rust, f"digest mismatch for {value!r}" + + +def test_rust_normalizes_one_and_one_point_zero_to_same(rust_probe): + a, b = _rust_digests(rust_probe, [{"x": 1}, {"x": 1.0}]) + assert a == b == cd._canonical_payload_digest({"x": 1}) + + +def _real_recording(): + return cert.st.recording_dir("vw", "single-cut-clojure-legacy") + + +@pytest.mark.parametrize("rel,pin", [ + ("py/step-000.json", "4b8ef78560ee2daa84cf4fd48da55b56ea902b2dd4edac4d04e9056c9582c83b"), + ("clj/step-000.blob.json", "acccb9e192e3cd762458d6b5e55fd6c2c1184d01fe3832c6c4e6da229029cb76"), +]) +def test_digest_parity_on_real_vw_blobs(rust_probe, rel, pin): + path = _real_recording() / rel + if not path.exists(): + pytest.skip(f"real recording {path} not present") + raw = path.read_bytes() + if hashlib.sha256(raw).hexdigest() != pin: + pytest.skip("retained blob differs from the pinned bytes") + obj = json.loads(raw) + (rust,) = _rust_digests(rust_probe, [obj]) + assert cd._canonical_payload_digest(obj) == rust + + +def test_real_py_blob_matches_astra_cited_rust_digest(rust_probe): + path = _real_recording() / "py/step-000.json" + if not path.exists(): + pytest.skip("real recording not present") + obj = json.loads(path.read_bytes()) + (rust,) = _rust_digests(rust_probe, [obj]) + # The digest Astra measured with the standalone store.rs probe (review r5). + assert rust == "b80bb10831a8de09558a9cdd4cef2c665368d674d13366f77874981bf30e6623" + assert cd._canonical_payload_digest(obj) == rust + + +# --------------------------------------------------------------------------- +# Round 7 (board [449]): integer boundary digest + fixture fail-vs-skip. +# --------------------------------------------------------------------------- +def test_u64_boundary_matches_astra_cited_digest(rust_probe): + """{'x': 2**64} takes serde's f64 representation before number(); the ported + digest must equal the value Astra measured with the standalone store.rs probe.""" + (rust,) = _rust_digests(rust_probe, [{"x": 2 ** 64}]) + assert rust == "eb369f44996329cb7252a3fad311df14e1e46bbc28f56aa15a9baa381ae16f8f" + assert cd._canonical_payload_digest({"x": 2 ** 64}) == rust + + +@pytest.mark.parametrize("value", [ + 2 ** 64, 2 ** 64 + 1, -(2 ** 63) - 1, 10 ** 25 + 1, 2 ** 53 + 1, 2 ** 63, + 2 ** 64 - 1, -(2 ** 63), +]) +def test_integer_boundary_digest_parity(rust_probe, value): + (rust,) = _rust_digests(rust_probe, [{"x": value}]) + assert cd._canonical_payload_digest({"x": value}) == rust, f"digest mismatch for {value}" + + +def test_probe_deps_pinned_to_coordinator_lock(rust_probe): + """The probe's serde/serde_json/zmij resolve to the coordinator's exact lock + entries (the digest is only as reproducible as its float formatter).""" + import tomllib + + def versions(lock_path): + pkgs = tomllib.loads(lock_path.read_text())["package"] + return {p["name"]: p["version"] for p in pkgs + if p["name"] in ("serde", "serde_json", "zmij")} + + probe_lock = rust_probe.parents[2] / "Cargo.lock" + coord_lock = _find_store_rs().parents[1] / "Cargo.lock" + assert versions(probe_lock) == versions(coord_lock) + + +def test_fixture_fails_not_skips_on_build_error(): + """Correction 3: with cargo present, a build failure must be a test FAILURE, + not a skip that hides a broken digest oracle.""" + from types import SimpleNamespace + from unittest.mock import patch + import tempfile + + store = _find_store_rs() + if store is None or _find_cargo() is None: + pytest.skip("prerequisites genuinely absent") + with tempfile.TemporaryDirectory(prefix="p045-r7-fixture-") as td: + factory = SimpleNamespace(mktemp=lambda _: Path(td)) + with patch(__name__ + "._find_cargo", return_value="/present/cargo"), \ + patch(__name__ + "._find_store_rs", return_value=store), \ + patch.object(subprocess, "run", + return_value=SimpleNamespace(returncode=101, stderr="error[E0425]")): + # A build failure must raise Failed (pytest.fail), NOT Skipped. + with pytest.raises(pytest.fail.Exception): + rust_probe.__wrapped__(factory) + with pytest.raises(BaseException) as exc: + rust_probe.__wrapped__(factory) + assert not isinstance(exc.value, pytest.skip.Exception) + + +# --------------------------------------------------------------------------- +# Round 9 (board [459]): valid Unicode parity; serde refuses lone surrogates. +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("value", [ + "é", chr(0x1F600), json.loads('"\\ud83d\\ude00"'), # value: BMP + astral + pair +]) +def test_valid_astral_value_digest_parity(rust_probe, value): + (rust,) = _rust_digests(rust_probe, [{"x": value}]) + assert cd._canonical_payload_digest({"x": value}) == rust + + +def test_astral_key_digest_parity(rust_probe): + obj = {json.loads('"\\ud83d\\ude00"'): 1, "astral \U0001d538 key": "✓"} + (rust,) = _rust_digests(rust_probe, [obj]) + assert cd._canonical_payload_digest(obj) == rust + + +def test_serde_refuses_lone_surrogate_like_the_bridge(rust_probe): + """A JSON text with an unpaired surrogate fails to parse into a Rust String; + the standalone probe errors on it (nonzero exit / no digest line), mirroring + the bridge's graded BridgeError.""" + proc = subprocess.run([str(rust_probe)], input='"\\ud800"\n', + capture_output=True, text=True) + assert proc.returncode != 0 or proc.stdout.strip() == "" + with pytest.raises(cd.BridgeError): + cd._canonical_payload_digest({"x": json.loads('"\\ud800"')}) diff --git a/delphi/tests/replay_harness/test_certify_rust_identity.py b/delphi/tests/replay_harness/test_certify_rust_identity.py new file mode 100644 index 0000000000..11cc5cf2d4 --- /dev/null +++ b/delphi/tests/replay_harness/test_certify_rust_identity.py @@ -0,0 +1,237 @@ +"""P-045 slice 2 (half): S1 closed, VERSIONED identity on a candidate checkpoint. + +Pure controls. Round 2 (board [398]) correction 3: the validator binds the +checkpoint schema and every retained field's expected value/type, rejects null +identity fields, empty admission strings and unknown admission keys. The real +DB-side identity is graded by the P-026 coordinator suite against Postgres. +""" + +from __future__ import annotations + +import pytest + +from polismath.replay import coordinator_driver as cd + + +def _cursors(): + return {"votes": {"slot": 0, "sha256": "a" * 64}, + "moderation": {"slot": 0, "sha256": "b" * 64}} + + +def _files(): + return {k: {"path": f"checkpoint-0/{k}.json", "bytes": 12, "sha256": "c" * 64} + for k in cd.S1_FILE_KEYS} + + +def _manifest(**over): + """The FULL 15-field polis-candidate-checkpoint/1 envelope emitted by + engine_adapter.snapshot (files + both cursor maps included).""" + admission = { + "candidate_schema": cd.S1_CANDIDATE_SCHEMA, + "engine_version": cd.S1_ENGINE_VERSION, + "input_digest": "sha256:i", "schedule_digest": "sha256:s", + "operation_id": "op-1", + } + admission.update(over.pop("admission", {})) + m = { + "schema": cd.S1_CHECKPOINT_SCHEMA, "admission": admission, + "protocol": cd.S1_PROTOCOL, "fixture_id": 1, "run_id": "r", + "session_id": "s", "compute_id": "compute-0", "checkpoint_id": "checkpoint-0", + "output_schema": cd.S1_OUTPUT_SCHEMA, "state_schema": cd.S1_STATE_SCHEMA, + "profile": cd.S1_PROFILE_WIRE, "persistence": False, + "math_input_cursors": _cursors(), "observed_state_cursors": _cursors(), + "files": _files(), + } + m.update(over) + return m + + +def test_identity_intact(): + assert cd.validate_s1_identity(_manifest()) == [] + + +def test_checkpoint_schema_required(): + assert any("schema" in f for f in cd.validate_s1_identity(_manifest(schema="other/1"))) + + +def test_all_null_checkpoint_fields_rejected(): + """Correction 3 reproduction, inverted: null retained fields + empty admission + strings + an extra admission key must NOT pass.""" + m = {k: None for k in cd.S1_CHECKPOINT_FIELDS} + m["schema"] = cd.S1_CHECKPOINT_SCHEMA + m["admission"] = dict(candidate_schema=cd.S1_CANDIDATE_SCHEMA, + engine_version=cd.S1_ENGINE_VERSION, input_digest="", + schedule_digest="", operation_id="", extra=True) + fails = cd.validate_s1_identity(m) + assert fails + assert any("unknown key" in f for f in fails) + assert any("nonempty string" in f for f in fails) + + +def test_wrong_candidate_schema_rejected(): + assert any("candidate_schema" in f for f in + cd.validate_s1_identity(_manifest(admission={"candidate_schema": "polis-candidate-input/2"}))) + + +def test_wrong_engine_version_is_not_accepted_as_s1(): + fails = cd.validate_s1_identity(_manifest(admission={"engine_version": "python-conversation/p026-s2"})) + assert any("engine_version" in f and "negotiate" in f for f in fails) + + +def test_unknown_admission_key_rejected(): + assert any("unknown key" in f for f in + cd.validate_s1_identity(_manifest(admission={"surprise": 1}))) + + +@pytest.mark.parametrize("field", cd.S1_ADMISSION_FIELDS) +def test_missing_admission_field_rejected(field): + m = _manifest() + m["admission"].pop(field) + assert any(field in f for f in cd.validate_s1_identity(m)) + + +@pytest.mark.parametrize("field", ["input_digest", "schedule_digest", "operation_id"]) +def test_empty_admission_digest_rejected(field): + assert any(field in f for f in cd.validate_s1_identity(_manifest(admission={field: ""}))) + + +@pytest.mark.parametrize("field", cd.S1_CHECKPOINT_FIELDS) +def test_missing_checkpoint_field_rejected(field): + m = _manifest() + m.pop(field) + assert any(field in f for f in cd.validate_s1_identity(m)) + + +@pytest.mark.parametrize("field,bad", [ + ("output_schema", "polis-candidate-math-output/2"), + ("state_schema", "rebuild-prefix/2"), + ("profile", "not-candidate"), + ("protocol", "polis-engine/2"), + ("fixture_id", True), # bool is not a plain int + ("fixture_id", "1"), + ("persistence", "no"), + ("persistence", True), # round 3: persistence must be EXACTLY False + ("run_id", ""), +]) +def test_bound_checkpoint_value_rejected(field, bad): + assert any(field in f for f in cd.validate_s1_identity(_manifest(**{field: bad}))) + + +# --------------------------------------------------------------------------- +# Round 3 correction 2: per-run expected admission/identity binding. +# --------------------------------------------------------------------------- +_EXPECTED_ADMISSION = {"input_digest": "sha256:i", "schedule_digest": "sha256:s", + "operation_id": "op-1"} +_EXPECTED_IDENTITY = {"run_id": "r", "session_id": "s", "compute_id": "compute-0", + "checkpoint_id": "checkpoint-0"} + + +def test_expected_binding_accepts_matching_identity(): + assert cd.validate_s1_identity(_manifest(), expected_admission=_EXPECTED_ADMISSION, + expected_identity=_EXPECTED_IDENTITY) == [] + + +def test_expected_binding_rejects_foreign_session(): + fails = cd.validate_s1_identity(_manifest(session_id="foreign-session"), + expected_identity=_EXPECTED_IDENTITY) + assert any("session_id" in f and "expected" in f for f in fails) + + +def test_expected_binding_rejects_changed_input_digest(): + fails = cd.validate_s1_identity(_manifest(admission={"input_digest": "sha256:CHANGED"}), + expected_admission=_EXPECTED_ADMISSION) + assert any("input_digest" in f and "expected" in f for f in fails) + + +# --------------------------------------------------------------------------- +# Round 4 correction 2: full worker envelope + fixture binding. +# --------------------------------------------------------------------------- +def test_full_envelope_positive_fixture_accepted(): + """The full 15-field emitted shape validates clean (positive control).""" + assert cd.validate_s1_identity(_manifest()) == [] + + +@pytest.mark.parametrize("field", ["files", "math_input_cursors", "observed_state_cursors"]) +def test_missing_envelope_field_rejected(field): + m = _manifest() + m.pop(field) + assert any("envelope" in f or field in f for f in cd.validate_s1_identity(m)) + + +def test_unknown_top_level_field_rejected(): + assert any("unknown key" in f for f in cd.validate_s1_identity(_manifest(surprise=True))) + + +@pytest.mark.parametrize("mutate", [ + lambda m: m["files"].pop("restore"), + lambda m: m["files"]["main"].pop("sha256"), + lambda m: m["files"]["main"].update(bytes="10"), # bytes must be a plain int + lambda m: m["math_input_cursors"]["votes"].update(slot=True), # boolean cursor slot + lambda m: m["observed_state_cursors"].pop("moderation"), +]) +def test_malformed_files_or_cursors_rejected(mutate): + m = _manifest() + mutate(m) + assert cd.validate_s1_identity(m) + + +def test_expected_fixture_binding_rejects_foreign_fixture(): + """Round 4: fixture_id is bound — a checkpoint for fixture 999 must not pass + when the caller expects fixture 1.""" + expected = dict(_EXPECTED_IDENTITY, fixture_id=1) + fails = cd.validate_s1_identity(_manifest(fixture_id=999), expected_identity=expected) + assert any("fixture_id" in f and "expected" in f for f in fails) + + +def test_missing_admission_block(): + fails = cd.validate_s1_identity({"schema": cd.S1_CHECKPOINT_SCHEMA}) + assert any("missing admission" in f for f in fails) + + +def test_campaign_cut_binding_distinguishes_plan_hash_from_schedule_digest(): + b = cd.bind_campaign_cut(compute_id="compute-0", checkpoint_id="checkpoint-0", + global_cut_index=5, schedule_digest="sha256:s1sched", + campaign_plan_sha256="sha256:campaign") + assert b["global_cut_index"] == 5 + assert b["s1_schedule_digest"] != b["campaign_plan_sha256"] + + +# --------------------------------------------------------------------------- +# Round 5 (board [440]): worker cursor/file custody bound by VALUE, not shape. +# --------------------------------------------------------------------------- +def test_input_and_observed_cursors_must_agree(): + m = _manifest() + m["observed_state_cursors"]["votes"]["slot"] = 999 # input stays 0 + fails = cd.validate_s1_identity(m, expected_admission=m["admission"], expected_identity=m) + assert any("disagree" in f for f in fails) + + +@pytest.mark.parametrize("mutate", [ + lambda m: m["files"]["main"].update(bytes=-1), # negative descriptor length + lambda m: m["files"]["main"].update(sha256="not-a-digest"), + lambda m: m["math_input_cursors"]["votes"].update(slot=-1), # negative cursor slot + lambda m: m["math_input_cursors"]["votes"].update(sha256="not-a-digest"), +]) +def test_negative_lengths_and_invalid_digests_rejected(mutate): + m = _manifest() + mutate(m) + assert cd.validate_s1_identity(m) + + +def test_expected_cursors_and_files_binding(): + m = _manifest() + good_cursors = _cursors() + assert cd.validate_s1_identity(m, expected_cursors=good_cursors) == [] + other = _cursors() + other["votes"]["slot"] = 7 + assert cd.validate_s1_identity(m, expected_cursors=other) + assert cd.validate_s1_identity(m, expected_files={"nope": 1}) + + +# --------------------------------------------------------------------------- +# Round 6 (board [445]): graded outer-container admission. +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("bad", [None, [], "x", 7]) +def test_validate_s1_identity_nonobject_is_graded(bad): + fails = cd.validate_s1_identity(bad) + assert isinstance(fails, list) and fails and all(isinstance(f, str) for f in fails) diff --git a/delphi/tests/replay_harness/test_certify_rust_publication.py b/delphi/tests/replay_harness/test_certify_rust_publication.py new file mode 100644 index 0000000000..1dde68f3de --- /dev/null +++ b/delphi/tests/replay_harness/test_certify_rust_publication.py @@ -0,0 +1,609 @@ +"""P-045 slice 2 (half): four-row readback + independent observer validators. + +Pure controls over synthetic readback bundles — no Postgres, no Rust binary. The +slice-3 N05/N10/N11 real-mutation controls (corrupt each table, sever a COMMIT, +substitute foreign-env rows) will drive THIS validator against a real Rust +publication; here it is graded on hand-built bundles and their mutations. + +Round 3 (board [418]) corrections 2 and 1: readback binds the full +input_checkpoint and types the epoch; the observer models the ACTUAL +math_writer.derive_bidtopid wrapper shape. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +from types import SimpleNamespace + +import pytest + +from polismath.poller.math_writer import derive_bidtopid +from polismath.replay import coordinator_driver as cd + + +def _canon(data): + # Delegate to the ported Rust storage_digest (no duplicate implementation). + return cd._canonical_payload_digest(data) + + +def _set(b, name, data): + raw = json.dumps(data) + dg = hashlib.sha256(raw.encode()).hexdigest() + b[name] = dict(b.get(name, {}), math_tick=b["ticks"]["math_tick"], data=data, + original_bytes=raw, original_sha256=dg) + b["ticks"]["original_digests"][name] = dg + ckpt = b["ticks"].get("input_checkpoint") + if isinstance(ckpt, dict): # keep the store checkpoint's digests in sync + if isinstance(ckpt.get("original_digests"), dict): + ckpt["original_digests"][name] = dg + if isinstance(ckpt.get("payload_digests"), dict): + ckpt["payload_digests"][name] = _canon(data) + + +def _bundle(tick=0, epoch=5, op="op-1"): + """A coherent, REAL-shaped bundle: main carries base-clusters and bidtopid is + the writer's {zid, bidToPid, lastVoteTimestamp} wrapper positionally aligned to + it (derive_bidtopid), plus a full polis-coordinator/1 store input_checkpoint + (schema, operation, epoch, original + canonical payload digests, cursors) — so + observer AND readback grade it clean.""" + conv = SimpleNamespace(base_clusters=[{"id": 2, "members": [1, 2]}, + {"id": 8, "members": [3, 4]}], last_updated=1000) + b = {"zid": 1, "math_env": "rustproto", + "ticks": {"math_tick": tick, "caching_tick": 42, "publisher_epoch": epoch, + "operation_id": op, "original_digests": {}}} + _set(b, "main", {"base-clusters": {"id": [2, 8], "members": [[1, 2], [3, 4]], "count": [2, 2]}}) + _set(b, "bidtopid", derive_bidtopid(conv, 1)) + _set(b, "ptptstats", {"1": {"a": 1}}) + b["main"]["caching_tick"] = 42 + b["ticks"]["input_checkpoint"] = { + "schema": "polis-coordinator/1", "operation_id": op, "publisher_epoch": epoch, + "original_digests": dict(b["ticks"]["original_digests"]), + "payload_digests": {n: _canon(b[n]["data"]) for n in ("main", "bidtopid", "ptptstats")}, + "cursors": {"votes": {"slot": 0, "sha256": "a" * 64}, + "moderation": {"slot": 0, "sha256": "b" * 64}}, + } + return b + + +def _V(b, **kw): + kw.setdefault("expected_prior_tick", None) + kw.setdefault("operation_id", "op-1") + kw.setdefault("publisher_epoch", 5) + return cd.validate_readback(b, **kw) + + +def test_first_publication_tick_zero_passes(): + assert _V(_bundle(tick=0)) == [] + + +def test_prior_to_next_tick(): + b = _bundle(tick=3) + assert _V(b, expected_prior_tick=2) == [] + assert any("prior+1" in f for f in _V(b, expected_prior_tick=5)) + + +def test_first_publication_must_be_zero(): + assert any("must be 0" in f for f in _V(_bundle(tick=1))) + + +def test_tick_disagreement_detected(): + b = _bundle(tick=0) + b["bidtopid"]["math_tick"] = 1 + assert any("bidtopid" in f for f in _V(b)) + + +def test_missing_companion_detected(): + b = _bundle(tick=0) + b["ptptstats"] = {} + assert any("ptptstats" in f for f in _V(b)) + + +def test_wrong_epoch_and_operation_detected(): + b = _bundle(tick=0) + fails = _V(b, operation_id="OTHER", publisher_epoch=99) + assert any("operation_id" in f for f in fails) and any("publisher_epoch" in f for f in fails) + + +def test_data_text_is_not_original_evidence(): + b = _bundle(tick=0) + b["main"].pop("original_bytes") + assert any("original_bytes absent" in f for f in _V(b)) + + +def test_original_byte_mutation_detected(): + b = _bundle(tick=0) + b["main"]["original_bytes"] = json.dumps({"base-clusters": {"id": [2, 8], "x": 0}}) + assert _V(b) + + +def test_jsonb_non_correspondence_detected(): + b = _bundle(tick=0) + b["main"]["data"] = {"base-clusters": {"id": [9, 9]}} # disagrees with original_bytes + assert any("correspond" in f for f in _V(b)) + + +def test_companion_caching_tick_rejected(): + b = _bundle(tick=0) + b["bidtopid"]["caching_tick"] = 3 + assert any("companion must not carry a caching_tick" in f for f in _V(b)) + + +def test_main_missing_caching_tick_rejected(): + b = _bundle(tick=0) + b["main"].pop("caching_tick") + assert any("caching_tick absent" in f for f in _V(b)) + + +# --------------------------------------------------------------------------- +# Round 2 correction 4 (kept): readback false accepts. +# --------------------------------------------------------------------------- +def test_missing_jsonb_evidence_rejected(): + b = _bundle(tick=0) + for name in ("main", "bidtopid", "ptptstats"): + del b[name]["data"] + assert _V(b) + + +def test_boolean_math_ticks_rejected(): + b = _bundle(tick=0) + for name in ("main", "bidtopid", "ptptstats", "ticks"): + b[name]["math_tick"] = False + assert any("non-boolean" in f for f in _V(b)) + + +def test_publication_scope_absent_rejected(): + b = _bundle(tick=0) + del b["zid"], b["math_env"] + fails = _V(b) + assert any("zid" in f for f in fails) and any("math_env" in f for f in fails) + + +def test_integer_vs_boolean_correspondence_rejected(): + b = _bundle(tick=0) + _set(b, "main", {"value": 1}) + b["main"]["caching_tick"] = 42 + b["main"]["data"] = {"value": True} + assert any("correspond" in f for f in _V(b)) + + +def test_expected_zid_and_env_binding(): + b = _bundle(tick=0) + assert _V(b, expected_zid=1, expected_math_env="rustproto") == [] + assert _V(b, expected_zid=999) + assert _V(b, expected_math_env="python") + + +def test_pg_numeric_normalization_still_allowed(): + b = _bundle(tick=0) + _set(b, "main", {"x": 1}) + b["main"]["caching_tick"] = 42 + b["main"]["data"] = {"x": 1.0} + b["ticks"]["input_checkpoint"]["payload_digests"]["main"] = _canon({"x": 1.0}) + assert _V(b) == [] + + +# --------------------------------------------------------------------------- +# Round 3 correction 2: epoch typing + full input_checkpoint custody. +# --------------------------------------------------------------------------- +def test_boolean_epoch_rejected_against_integer_one(): + b = _bundle(tick=0) + b["ticks"]["publisher_epoch"] = True + assert any("publisher_epoch" in f for f in _V(b, publisher_epoch=1)) + + +def test_foreign_input_checkpoint_rejected(): + b = _bundle(tick=0) + b["ticks"]["input_checkpoint"] = {"foreign": True} + assert any("input_checkpoint" in f for f in _V(b)) + + +def test_coherent_input_checkpoint_accepted_and_bound(): + b = _bundle(tick=0) # already carries a full valid store checkpoint + ckpt = b["ticks"]["input_checkpoint"] + assert _V(b) == [] + assert _V(b, expected_input_checkpoint=dict(ckpt)) == [] + assert _V(b, expected_input_checkpoint=dict(ckpt, operation_id="OTHER")) + + +def test_input_checkpoint_operation_mismatch_rejected(): + b = _bundle(tick=0) + b["ticks"]["input_checkpoint"]["operation_id"] = "WRONG" + assert any("operation_id" in f for f in _V(b)) + + +# --------------------------------------------------------------------------- +# Round 4 correction 2: persisted store-checkpoint custody + typed cursors. +# --------------------------------------------------------------------------- +def test_absent_input_checkpoint_rejected(): + b = _bundle(tick=0) + del b["ticks"]["input_checkpoint"] + assert any("input_checkpoint" in f and "required" in f for f in _V(b)) + + +def test_operation_only_checkpoint_rejected(): + b = _bundle(tick=0) + b["ticks"]["input_checkpoint"] = {"operation_id": "op-1"} + assert _V(b) # missing schema/original_digests/cursors + + +def test_wrong_store_checkpoint_schema_rejected(): + b = _bundle(tick=0) + b["ticks"]["input_checkpoint"]["schema"] = "polis-candidate-checkpoint/1" + assert any("input_checkpoint.schema" in f for f in _V(b)) + + +def test_boolean_cursor_slot_rejected(): + """A boolean cursor slot must not equal an expected integer checkpoint.""" + b = _bundle(tick=0) + expected = {k: (dict(v) if isinstance(v, dict) else v) + for k, v in b["ticks"]["input_checkpoint"].items()} + expected["cursors"] = {"votes": {"slot": 1}, "moderation": {"slot": 0}} + b["ticks"]["input_checkpoint"]["cursors"]["votes"]["slot"] = True + # rejected both by cursor typing and by type-aware expected equality + assert _V(b) + assert _V(b, expected_input_checkpoint=expected) + + +def test_input_checkpoint_original_digests_mismatch_rejected(): + b = _bundle(tick=0) + b["ticks"]["input_checkpoint"]["original_digests"] = {"main": "0" * 64} + assert any("original_digests" in f for f in _V(b)) + + +# --------------------------------------------------------------------------- +# Round 3 correction 1: observer models the ACTUAL derive_bidtopid wrapper. +# --------------------------------------------------------------------------- +def test_observer_accepts_real_writer_wrapper(): + assert cd.observe_bundle_coherence(_bundle(tick=0)) == [] + + +def test_observer_accepts_valid_empty_generation(): + b = _bundle(tick=0) + _set(b, "main", {"base-clusters": {"id": [], "members": [], "count": []}}) + b["main"]["caching_tick"] = 42 + _set(b, "bidtopid", derive_bidtopid(SimpleNamespace(base_clusters=[], last_updated=0), 1)) + assert cd.observe_bundle_coherence(b) == [] + + +def test_observer_rejects_unrelated_invented_map(): + b = _bundle(tick=0) + _set(b, "bidtopid", {"2": [999], "8": [1]}) + fails = cd.observe_bundle_coherence(b) + assert fails and any("bidtopid" in f for f in fails) + + +def _real_bundle_with(main_base, groups=None): + """A bundle whose main.base-clusters is `main_base` and whose bidtopid wrapper + is positionally derived from the same members — so a shape/positional check + passes and only the membership INVARIANTS decide the verdict.""" + b = _bundle(tick=0) + main = {"base-clusters": main_base} + if groups is not None: + main["group-clusters"] = groups + ids, members = main_base["id"], main_base["members"] + conv = SimpleNamespace(base_clusters=[{"id": i, "members": m} for i, m in zip(ids, members)], + last_updated=1000) + _set(b, "main", main) + b["main"]["caching_tick"] = 42 + _set(b, "bidtopid", derive_bidtopid(conv, 1)) + return b + + +def test_observer_rejects_duplicate_and_overlapping_real_buckets(): + """Round 4 correction 1, on the REAL wrapper: `[[1,1],[1,4]]` has an intra- + bucket duplicate AND a participant (1) in two buckets.""" + b = _real_bundle_with({"id": [2, 8], "members": [[1, 1], [1, 4]], "count": [2, 2]}) + assert pub_reject(b) + + +def test_observer_rejects_wrong_base_counts(): + b = _real_bundle_with({"id": [2, 8], "members": [[1, 2], [3, 4]], "count": [999, 0]}) + assert any("count" in f for f in cd.observe_bundle_coherence(b)) + + +def test_observer_rejects_duplicate_base_ids(): + b = _real_bundle_with({"id": [2, 2], "members": [[1, 2], [3, 4]], "count": [2, 2]}) + assert any("base-clusters.id has duplicates" in f for f in cd.observe_bundle_coherence(b)) + + +def test_observer_rejects_unknown_group_bid(): + b = _real_bundle_with({"id": [2, 8], "members": [[1, 2], [3, 4]], "count": [2, 2]}, + groups=[{"id": 0, "members": [999]}]) + assert any("unknown base bid" in f for f in cd.observe_bundle_coherence(b)) + + +def test_observer_accepts_valid_group_reference(): + b = _real_bundle_with({"id": [2, 8], "members": [[1, 2], [3, 4]], "count": [2, 2]}, + groups=[{"id": 0, "members": [2, 8]}]) + assert cd.observe_bundle_coherence(b) == [] + + +def pub_reject(b): + return cd.observe_bundle_coherence(b) != [] + + +def test_observer_rejects_positional_membership_mismatch(): + b = _bundle(tick=0) + _set(b, "bidtopid", {"zid": 1, "bidToPid": [[9, 9], [3, 4]], "lastVoteTimestamp": 1000}) + assert any("positional bid membership mismatch" in f for f in cd.observe_bundle_coherence(b)) + + +def test_observer_rejects_bidtopid_zid_mismatch(): + b = _bundle(tick=0) + d = derive_bidtopid(SimpleNamespace(base_clusters=[{"id": 2, "members": [1, 2]}, + {"id": 8, "members": [3, 4]}], last_updated=1000), 999) + _set(b, "bidtopid", d) + assert any("zid" in f for f in cd.observe_bundle_coherence(b)) + + +def test_observer_rejects_absent_bundle(): + assert cd.observe_bundle_coherence({}) + + +def test_observer_detects_mixed_generation(): + b = _bundle(tick=0) + b["bidtopid"]["math_tick"] = 1 + assert any("different generation" in f for f in cd.observe_bundle_coherence(b)) + + +def test_observer_requires_row_presence(): + b = _bundle(tick=0) + del b["ptptstats"] + assert any("ptptstats" in f for f in cd.observe_bundle_coherence(b)) + + +def test_observer_injected_fold_runs(): + b = _bundle(tick=0) + calls = {} + + def fold(main): + calls["main"] = main + return ["fold: synthetic finding"] + + fails = cd.observe_bundle_coherence(b, fold_check=fold) + assert calls and any("synthetic finding" in f for f in fails) + + +def test_observer_fold_exception_is_a_finding_not_a_crash(): + def fold(_main): + raise RuntimeError("boom") + + assert any("boom" in f for f in cd.observe_bundle_coherence(_bundle(tick=0), fold_check=fold)) + + +def test_bundle_mutations_are_independent(): + a, b = _bundle(), _bundle() + a["main"]["math_tick"] = 99 + assert b["main"]["math_tick"] == 0 + assert copy.deepcopy(a) == a + + +# --------------------------------------------------------------------------- +# Round 5 (board [440]): total membership admission — graded, never a TypeError. +# --------------------------------------------------------------------------- +def _obs_reject(main): + b = _bundle(tick=0) + _set(b, "main", main) + assert _V(b) == [], "byte custody must still pass" + return cd.observe_bundle_coherence(b) + + +def _main_base(**over): + base = {"id": [2, 8], "members": [[1, 2], [3, 4]], "count": [2, 2]} + base.update(over) + return {"base-clusters": base} + + +def test_observer_rejects_boolean_main_pid(): + # main members [True,2] must not alias companion integer [1,2] via 1==True + m = _main_base(members=[[True, 2], [3, 4]]) + assert _obs_reject(m) + + +@pytest.mark.parametrize("groups", [ + [{"id": 0, "members": [2.0]}], # float bid aliases integer base id + {}, # wrong container (not a list) + [7], # scalar group row (not an object) +]) +def test_observer_rejects_malformed_groups(groups): + m = _main_base() + m["group-clusters"] = groups + assert _obs_reject(m) + + +@pytest.mark.parametrize("mutate", [ + lambda m: m["base-clusters"].__setitem__("members", [7, [3, 4]]), # scalar membership (was TypeError) + lambda m: m.__setitem__("group-clusters", [{"id": 0, "members": [[]]}]), # unhashable bid (was TypeError) +]) +def test_observer_total_admission_never_raises(mutate): + m = _main_base() + mutate(m) + # must be a graded rejection, not an exception + fails = _obs_reject(m) + assert fails + + +# --------------------------------------------------------------------------- +# Round 5: persisted checkpoint epoch / canonical-digest / cursor binding. +# --------------------------------------------------------------------------- +def test_checkpoint_missing_epoch_or_payload_digests_rejected(): + for field in ("publisher_epoch", "payload_digests"): + b = _bundle(tick=0) + b["ticks"]["input_checkpoint"].pop(field) + assert _V(b, expected_input_checkpoint=copy.deepcopy(b["ticks"]["input_checkpoint"])), field + + +def test_checkpoint_foreign_embedded_epoch_rejected(): + b = _bundle(tick=0) + b["ticks"]["input_checkpoint"]["publisher_epoch"] = 999 + assert any("publisher_epoch" in f for f in _V(b, expected_input_checkpoint=copy.deepcopy(b["ticks"]["input_checkpoint"]))) + + +def test_checkpoint_wrong_payload_digests_rejected(): + b = _bundle(tick=0) + b["ticks"]["input_checkpoint"]["payload_digests"] = {"main": "bad"} + assert any("payload_digests" in f for f in _V(b, expected_input_checkpoint=copy.deepcopy(b["ticks"]["input_checkpoint"]))) + + +def test_checkpoint_unknown_cursor_stream_rejected(): + b = _bundle(tick=0) + b["ticks"]["input_checkpoint"]["cursors"] = {"unrelated": {"slot": 0}} + assert _V(b, expected_input_checkpoint=copy.deepcopy(b["ticks"]["input_checkpoint"])) + + +# --------------------------------------------------------------------------- +# Round 6 (board [445]): graded outer-container admission; store digest / cursors. +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("bad", [None, [], "x", 7]) +def test_readback_nonobject_bundle_is_graded(bad): + fails = cd.validate_readback(bad, expected_prior_tick=None, operation_id="op-1", publisher_epoch=5) + assert isinstance(fails, list) and fails and all(isinstance(f, str) for f in fails) + + +@pytest.mark.parametrize("bad", [None, [], "x", 7]) +def test_observer_nonobject_bundle_is_graded(bad): + fails = cd.observe_bundle_coherence(bad) + assert isinstance(fails, list) and fails + + +def test_readback_nonobject_row_is_graded(): + fails = _V(dict(_bundle(), main=[1])) # a list-valued row must not crash + assert fails and all(isinstance(f, str) for f in fails) + + +def test_observer_nonobject_row_is_graded(): + assert cd.observe_bundle_coherence(dict(_bundle(), main=[1])) + + +@pytest.mark.parametrize("obj", [{"x": 1.0}, {"x": -0.0}, {"x": 1e-7}, {"x": "é"}]) +def test_store_bound_normalized_digest_accepted(obj): + """The ported digest normalizes PG-equal numbers and Unicode exactly like the + Rust store, so a store-bound checkpoint digest for a normalized payload is + ACCEPTED (was rejected under compact-JSON hashing).""" + b = _bundle(tick=0) + _set(b, "main", obj) + b["main"]["caching_tick"] = 42 + assert _V(b) == [] + # the checkpoint digest equals the ported (store) digest + assert b["ticks"]["input_checkpoint"]["payload_digests"]["main"] == cd._canonical_payload_digest(obj) + + +def test_digest_normalizes_one_and_one_point_zero_and_negative_zero(): + assert cd._canonical_payload_digest({"x": 1}) == cd._canonical_payload_digest({"x": 1.0}) + assert cd._canonical_payload_digest({"x": 0}) == cd._canonical_payload_digest({"x": -0.0}) + # ...but distinct from a genuinely different number + assert cd._canonical_payload_digest({"x": 1}) != cd._canonical_payload_digest({"x": 2}) + + +@pytest.mark.parametrize("mutate", [ + lambda c: c["cursors"]["votes"].pop("sha256"), # missing stored cursor hash + lambda c: c["cursors"]["votes"].__setitem__("sha256", []), # malformed + lambda c: c["cursors"]["votes"].__setitem__("sha256", "x" * 63), # not 64-hex +]) +def test_store_cursor_hash_required_and_typed(mutate): + b = _bundle(tick=0) + mutate(b["ticks"]["input_checkpoint"]) + assert _V(b, expected_input_checkpoint=copy.deepcopy(b["ticks"]["input_checkpoint"])) + + +# --------------------------------------------------------------------------- +# Round 7 (board [449]): every remaining row dereference is graded, not raised. +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("row,value", [ + ("ticks", [1]), ("bidtopid", [1]), ("ptptstats", [1]), ("main", 7), + ("ticks", 7), ("bidtopid", "x"), ("ptptstats", None), +]) +def test_readback_malformed_row_is_graded_not_raised(row, value): + b = _bundle() + b[row] = value + fails = _V(b) # must not raise + assert isinstance(fails, list) and fails and all(isinstance(f, str) for f in fails) + + +@pytest.mark.parametrize("row,value", [ + ("ticks", [1]), ("bidtopid", [1]), ("ptptstats", [1]), ("main", 7), +]) +def test_observer_malformed_row_is_graded_not_raised(row, value): + b = _bundle() + b[row] = value + assert cd.observe_bundle_coherence(b) # graded, no exception + + +def test_integer_boundary_digest_is_serde_dispatched(): + """Python ints beyond i64/u64 take serde's f64 representation, so 2**64 and + 2**64+1 (both f64 -> same value) share a digest distinct from the exact int + string; u64::MAX and i64::MIN stay exact.""" + assert cd._canonical_payload_digest({"x": 2 ** 64}) == cd._canonical_payload_digest({"x": 2 ** 64 + 1}) + assert cd._canonical_payload_digest({"x": 2 ** 64 - 1}) != cd._canonical_payload_digest({"x": 2 ** 64}) + # exact integers in range are byte-for-byte their decimal spelling + assert cd._canonical_payload_digest(2 ** 64 - 1) != cd._canonical_payload_digest(2 ** 64) + + +# --------------------------------------------------------------------------- +# Round 8 (board [454]): custody values admitted before use; digest error graded. +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("change", [ + lambda b: b["ticks"].update(original_digests=[1]), # was AttributeError + lambda b: b["main"].update(original_bytes=[[]]), # was TypeError + lambda b: b["main"].update(original_bytes={"x": 1}), # was TypeError + lambda b: b["main"].update(data={"x": 10 ** 400}), # was BridgeError escape +]) +def test_readback_custody_failures_are_graded_not_raised(change): + b = _bundle() + change(b) + fails = _V(b) # must not raise + assert isinstance(fails, list) and fails and all(isinstance(f, str) for f in fails) + + +def test_out_of_range_payload_integer_is_graded_in_readback(): + b = _bundle() + b["main"]["data"] = {"x": 10 ** 400} + assert any("canonical-digest" in f for f in _V(b)) + + +def test_low_level_overflow_still_raises_named_bridgeerror(): + with pytest.raises(cd.BridgeError): + cd._canonical_payload_digest({"x": 10 ** 400}) + + +def test_string_and_byte_original_evidence_still_accepted(): + b = _bundle() + assert _V(b) == [] # str original_bytes (factory default) + b2 = _bundle() + for name in ("main", "bidtopid", "ptptstats"): + b2[name]["original_bytes"] = b2[name]["original_bytes"].encode() # bytes evidence + assert _V(b2) == [] + + +# --------------------------------------------------------------------------- +# Round 9 (board [459]): unpaired surrogates graded; valid Unicode preserved. +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("change", [ + lambda b: b["main"].update(original_bytes=json.loads('"\\ud800"')), # lone high in original str + lambda b: b["main"].update(data=json.loads('{"x":"\\ud800"}')), # lone high in a value + lambda b: b["main"].update(data=json.loads('{"\\udfff":1}')), # lone low in a key +]) +def test_unpaired_surrogates_are_graded_not_raised(change): + b = _bundle() + change(b) + fails = _V(b) # must not raise UnicodeEncodeError + assert isinstance(fails, list) and fails and all(isinstance(f, str) for f in fails) + + +def test_lone_surrogate_digest_raises_named_bridgeerror(): + with pytest.raises(cd.BridgeError): + cd._canonical_payload_digest({"x": json.loads('"\\ud800"')}) + with pytest.raises(cd.BridgeError): + cd._canonical_payload_digest({json.loads('"\\udfff"'): 1}) + + +@pytest.mark.parametrize("value", ["é", chr(0x1F600), json.loads('"\\ud83d\\ude00"')]) +def test_valid_non_bmp_unicode_still_digests(value): + # a proper surrogate pair is one astral scalar to Python's JSON parser + assert cd._canonical_payload_digest({"x": value}) # no raise, deterministic + b = _bundle() + _set(b, "main", {"x": value}) + b["main"]["caching_tick"] = 42 + assert _V(b) == [] diff --git a/delphi/tests/replay_harness/test_certify_rust_stages.py b/delphi/tests/replay_harness/test_certify_rust_stages.py new file mode 100644 index 0000000000..3234c80675 --- /dev/null +++ b/delphi/tests/replay_harness/test_certify_rust_stages.py @@ -0,0 +1,205 @@ +"""P-045 slice 2 (half): same-invocation stage sibling and stage-context binding. + +Pure controls — no Postgres, no Rust binary. They pin the bridge's stage +discipline: ONE IMMUTABLE stage sibling per operation (never truncate-and-replace, +never the write_stage_documents singleton-into-a-shared-dir data-loss path), the +py stage-engine label, the closed p045-stage-binding/1 sibling manifest, and a +plan/session-scoped stage context that rejects duplicates and foreign/mismatched +identities. Round 2 (board [398]) corrections 6 and 7 are pinned here. +""" + +from __future__ import annotations + +import json + +import pytest + +from polismath.replay import coordinator_driver as cd +from polismath.replay import stages + + +def test_capture_writes_immutable_per_operation_dirs(tmp_path, monkeypatch): + seen = {} + + def fake_doc(conv, *, step_index, digest, tick=None, blob=None, engine=stages.PY_STAGE_ENGINE): + seen["engine"] = engine + return {"schema": stages.STAGE_DUMP_SCHEMA, "engine": engine, "step": step_index, + "input_digest": digest, "tick": tick, "stages": {}} + + monkeypatch.setattr(stages, "stage_document", fake_doc) + p0 = cd.capture_rust_stage_sibling(object(), {"m": 1}, step_index=0, + semantic_input_digest="sha256:aa", + out_dir=tmp_path / "op-000", run_root=tmp_path) + p1 = cd.capture_rust_stage_sibling(object(), {"m": 2}, step_index=1, + semantic_input_digest="sha256:bb", + out_dir=tmp_path / "op-001", run_root=tmp_path) + assert p0.exists() and p1.exists() and p0 != p1 + assert seen["engine"] == "py" + assert json.loads(p0.read_text())["input_digest"] == "sha256:aa" + + +def test_capture_refuses_to_overwrite_immutable_evidence(tmp_path, monkeypatch): + """Correction 7: a second snapshot for the SAME operation/cut with different + content must be refused, not silently truncate-and-replace the first. An + identical retry is idempotent.""" + monkeypatch.setattr(stages, "stage_document", + lambda *a, **k: {"step": k["step_index"], "input_digest": k["digest"]}) + op = tmp_path / "op" + first = cd.capture_rust_stage_sibling(None, {}, step_index=0, + semantic_input_digest="a" * 64, out_dir=op, run_root=tmp_path) + original = first.read_bytes() + # Identical retry: idempotent, bytes preserved. + again = cd.capture_rust_stage_sibling(None, {}, step_index=0, + semantic_input_digest="a" * 64, out_dir=op, run_root=tmp_path) + assert again == first and first.read_bytes() == original + # Differing content for the same operation/cut: refused. + with pytest.raises(cd.BridgeError): + cd.capture_rust_stage_sibling(None, {}, step_index=0, + semantic_input_digest="b" * 64, out_dir=op, run_root=tmp_path) + assert first.read_bytes() == original, "existing evidence must be untouched" + + +def test_capture_confines_to_run_root(tmp_path, monkeypatch): + monkeypatch.setattr(stages, "stage_document", lambda *a, **k: {"step": k["step_index"]}) + with pytest.raises(cd.BridgeError): + cd.capture_rust_stage_sibling(None, {}, step_index=0, semantic_input_digest="a" * 64, + out_dir=tmp_path / "escape", run_root=tmp_path / "root") + + +def test_capture_does_not_call_the_deleting_writer(tmp_path, monkeypatch): + monkeypatch.setattr(stages, "stage_document", + lambda *a, **k: {"step": k["step_index"], "input_digest": k["digest"]}) + monkeypatch.setattr(stages, "write_stage_documents", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("must not delete siblings"))) + monkeypatch.setattr(stages, "run_stage_dump", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("must not re-replay"))) + cd.capture_rust_stage_sibling(None, {}, step_index=0, + semantic_input_digest="sha256:aa", out_dir=tmp_path / "op", run_root=tmp_path) + + +def test_stage_binding_manifest_shape(): + m = cd.stage_binding_manifest( + run_id="r", session_id="s", compute_id="c", checkpoint_id="k", + operation_id="op", input_digest="i", schedule_digest="sd", + semantic_input_digest="sha256:aa", stage_file_sha256="f", + output_file_sha256s={"main": "m", "bidtopid": "b", "ptptstats": "p"}, + binary_sha256="bin", worker_tree_sha256="wrk", profile=cd.PROFILE_SNAPSHOT_REBUILD) + assert m["schema"] == "p045-stage-binding/1" + assert m["producer"] == "rust" and m["stage_engine"] == "py" + assert m["profile"] == "snapshot-rebuild/1" + + +# --------------------------------------------------------------------------- +# Stage context (correction 6): full-identity keys, duplicate rejection. +# --------------------------------------------------------------------------- +def _entry(**over): + e = dict(plan_sha256="plan-1", run_id="r", session_id="sess-1", + compute_id="compute-0", checkpoint_id="checkpoint-0", + global_cut_index=0, semantic_input_digest="a" * 64, + profile=cd.PROFILE_SNAPSHOT_REBUILD) + e.update(over) + return e + + +def _ctx_file(tmp_path, entries, schema=cd.STAGE_CONTEXT_SCHEMA): + p = tmp_path / "ctx.json" + p.write_text(json.dumps({"schema": schema, "entries": entries})) + return p + + +def test_load_stage_context_rejects_wrong_schema(tmp_path): + with pytest.raises(cd.BridgeError): + cd.load_stage_context(_ctx_file(tmp_path, [], schema="p045-stage-context/2")) + + +def test_load_stage_context_rejects_duplicate_local_identity(tmp_path): + """Correction 6: two fresh S1 sessions reuse compute-0/checkpoint-0; keyed on + the FULL identity a duplicate is rejected, never last-writer-wins.""" + dup = _ctx_file(tmp_path, [_entry(global_cut_index=0), _entry(global_cut_index=1)]) + with pytest.raises(cd.BridgeError): + cd.load_stage_context(dup) + + +def test_load_stage_context_distinct_sessions_coexist(tmp_path): + two = _ctx_file(tmp_path, [_entry(session_id="sess-1", global_cut_index=0), + _entry(session_id="sess-2", global_cut_index=1)]) + ctx = cd.load_stage_context(two) + assert len(ctx) == 2 + + +@pytest.mark.parametrize("bad", [ + {"global_cut_index": -1}, + {"global_cut_index": True}, + {"profile": "battery-chain/2"}, + {"semantic_input_digest": ""}, +]) +def test_load_stage_context_rejects_bad_entry(tmp_path, bad): + with pytest.raises(cd.BridgeError): + cd.load_stage_context(_ctx_file(tmp_path, [_entry(**bad)])) + + +def test_load_stage_context_requires_full_identity(tmp_path): + partial = {"compute_id": "compute-0", "checkpoint_id": "checkpoint-0", + "global_cut_index": 0, "semantic_input_digest": "a" * 64, + "profile": cd.PROFILE_SNAPSHOT_REBUILD} # no plan/run/session + with pytest.raises(cd.BridgeError): + cd.load_stage_context(_ctx_file(tmp_path, [partial])) + + +def test_check_stage_context_ok_foreign_and_mismatch(tmp_path): + ctx = cd.load_stage_context(_ctx_file(tmp_path, [_entry()])) + common = dict(plan_sha256="plan-1", run_id="r", session_id="sess-1", + compute_id="compute-0", checkpoint_id="checkpoint-0") + assert cd.check_stage_context(ctx, derived_semantic_digest="a" * 64, + expected_profile=cd.PROFILE_SNAPSHOT_REBUILD, + expected_cut_index=0, **common) == [] + # foreign session + assert cd.check_stage_context(ctx, derived_semantic_digest="a" * 64, + **dict(common, session_id="sess-X")) + # digest mismatch + assert cd.check_stage_context(ctx, derived_semantic_digest="b" * 64, **common) + # wrong profile / cut + assert cd.check_stage_context(ctx, derived_semantic_digest="a" * 64, + expected_profile=cd.PROFILE_BATTERY_CHAIN, **common) + assert cd.check_stage_context(ctx, derived_semantic_digest="a" * 64, + expected_cut_index=7, **common) + + +# --------------------------------------------------------------------------- +# Round 3 correction 3: typed identity, run binding, graded malformed. +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("field", cd._STAGE_CONTEXT_IDENTITY) +def test_null_stage_identity_rejected(tmp_path, field): + with pytest.raises(cd.BridgeError): + cd.load_stage_context(_ctx_file(tmp_path, [_entry(**{field: None})])) + + +def test_malformed_list_identity_is_graded_not_typeerror(tmp_path): + """A list-valued session must raise a graded BridgeError, never an ungraded + TypeError during key construction.""" + with pytest.raises(cd.BridgeError): + cd.load_stage_context(_ctx_file(tmp_path, [_entry(session_id=[])])) + + +def test_unknown_stage_context_key_rejected(tmp_path): + with pytest.raises(cd.BridgeError): + cd.load_stage_context(_ctx_file(tmp_path, [dict(_entry(), surprise=1)])) + + +def test_check_stage_context_binds_run(tmp_path): + """A foreign run with the same local session/compute/checkpoint/plan must NOT + match, and a consumer that omits run_id fails closed.""" + ctx = cd.load_stage_context(_ctx_file(tmp_path, [_entry(run_id="foreign-run")])) + addr = dict(plan_sha256="plan-1", session_id="sess-1", compute_id="compute-0", + checkpoint_id="checkpoint-0", derived_semantic_digest="a" * 64) + # foreign run -> not found + assert cd.check_stage_context(ctx, run_id="r", **addr) + # right run -> found and clean + assert cd.check_stage_context(ctx, run_id="foreign-run", **addr) == [] + # no run_id -> fails closed + assert any("run_id" in f for f in cd.check_stage_context(ctx, **addr)) + + +def test_distinct_runs_coexist(tmp_path): + two = _ctx_file(tmp_path, [_entry(run_id="run-1"), _entry(run_id="run-2")]) + assert len(cd.load_stage_context(two)) == 2