diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7da5f19f..b8d81477 100755 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -35,3 +35,30 @@ jobs: run: make dev-install - name: cargo test run: cargo test --all-features --all-targets --benches + + integration-real: + name: Integration tests (real nodes) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Install protoc + run: sudo apt install -y protobuf-compiler + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.85.0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Read pinned ldk-server rev + id: ldk-rev + run: echo "rev=$(grep -o 'ldk-server?rev=[a-f0-9]*' Cargo.lock | head -1 | cut -d= -f2)" >> "$GITHUB_OUTPUT" + - name: Build ldk-server image + uses: docker/build-push-action@v6 + with: + context: "https://github.com/lightningdevkit/ldk-server.git#${{ steps.ldk-rev.outputs.rev }}" + tags: simln-ldk-server:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Run real-node integration tests + run: make integration-real + env: + SIMLN_LDK_SERVER_IMAGE: simln-ldk-server:ci diff --git a/Cargo.lock b/Cargo.lock index dfc54a32..ec3a7876 100755 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstyle" version = "1.0.10" @@ -132,7 +141,7 @@ dependencies = [ "http-body 0.4.6", "hyper 0.14.32", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", @@ -159,7 +168,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", @@ -172,6 +181,31 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core 0.5.6", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper 1.0.2", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.3.4" @@ -209,6 +243,24 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper 1.0.2", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -294,6 +346,32 @@ dependencies = [ "hex-conservative 0.2.2", ] +[[package]] +name = "bitcoincore-rpc" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6c0ee9354e3dac217db4cb1dd31941073a87fe53c86bcf3eb2b8bc97f00a08" +dependencies = [ + "bitcoin-private", + "bitcoincore-rpc-json", + "jsonrpc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bitcoincore-rpc-json" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d30ce6f40fb0a2e8d98522796219282504b7a4b14e2b4c26139a7bea6aec6586" +dependencies = [ + "bitcoin", + "bitcoin-private", + "serde", + "serde_json", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -302,9 +380,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "blake2b_simd" @@ -317,6 +395,82 @@ dependencies = [ "constant_time_eq", ] +[[package]] +name = "bollard" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899ca34eb6924d6ec2a77c6f7f5c7339e60fd68235eaf91edd5a15f12958bb06" +dependencies = [ + "async-stream", + "base64 0.22.1", + "bitflags 2.13.1", + "bollard-buildkit-proto", + "bollard-stubs", + "bytes", + "chrono", + "futures-core", + "futures-util", + "hex", + "home", + "http 1.3.1", + "http-body-util", + "hyper 1.6.0", + "hyper-named-pipe", + "hyper-rustls 0.27.5", + "hyper-util", + "hyperlocal", + "log", + "num", + "pin-project-lite", + "rand 0.9.5", + "rustls 0.23.25", + "rustls-native-certs", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror 2.0.19", + "tokio", + "tokio-stream", + "tokio-util", + "tonic 0.13.1", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-buildkit-proto" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b3e79f8bd0f25f32660e3402afca46fd91bebaf135af017326d905651f8107" +dependencies = [ + "prost 0.13.5", + "prost-types 0.13.5", + "tonic 0.13.1", + "ureq", +] + +[[package]] +name = "bollard-stubs" +version = "1.48.3-rc.28.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ea257e555d16a2c01e5593f40b73865cdf12efbceda33c6d14a2d8d1490368" +dependencies = [ + "base64 0.22.1", + "bollard-buildkit-proto", + "bytes", + "chrono", + "prost 0.13.5", + "serde", + "serde_json", + "serde_repr", + "serde_with", +] + [[package]] name = "bumpalo" version = "3.17.0" @@ -356,6 +510,18 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link 0.2.1", +] + [[package]] name = "clap" version = "4.5.34" @@ -493,6 +659,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -563,6 +739,41 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.100", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.100", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -588,11 +799,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -604,7 +816,7 @@ dependencies = [ "console", "shell-words", "tempfile", - "thiserror", + "thiserror 1.0.69", "zeroize", ] @@ -630,6 +842,17 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "docker_credential" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" +dependencies = [ + "base64 0.22.1", + "serde", + "serde_json", +] + [[package]] name = "downcast" version = "0.11.0" @@ -673,6 +896,17 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "etcetera" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c7b13d0780cb82722fd59f6f57f925e143427e4a75313a6c77243bf5326ae6" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.59.0", +] + [[package]] name = "expanduser" version = "1.2.2" @@ -710,6 +944,16 @@ dependencies = [ "tower 0.4.13", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -839,6 +1083,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.31" @@ -897,6 +1147,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "h2" version = "0.3.26" @@ -1136,6 +1392,20 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" +dependencies = [ + "hex", + "hyper 1.6.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -1227,6 +1497,45 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.6.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -1345,6 +1654,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.0.3" @@ -1374,6 +1689,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -1384,6 +1700,33 @@ checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" dependencies = [ "equivalent", "hashbrown 0.15.2", + "serde", +] + +[[package]] +name = "integration-tests" +version = "0.1.0" +dependencies = [ + "anyhow", + "bitcoin", + "bitcoincore-rpc", + "cln-grpc", + "csv", + "fedimint-tonic-lnd", + "hex", + "ldk-server-client", + "log", + "ntest", + "reqwest 0.12.15", + "rstest", + "serde_json", + "sim-cli", + "simln-lib", + "tempfile", + "testcontainers", + "tokio", + "tokio-util", + "tonic 0.8.3", ] [[package]] @@ -1435,6 +1778,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonrpc" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8128f36b47411cd3f044be8c1f5cc0c9e24d1d1bfdc45f0a57897b32513053f2" +dependencies = [ + "base64 0.13.1", + "serde", + "serde_json", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1541,6 +1895,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.7.4" @@ -1628,7 +1988,7 @@ dependencies = [ "log", "parking_lot", "queue-ext", - "rand", + "rand 0.8.5", "std-ext", ] @@ -1653,10 +2013,10 @@ dependencies = [ "libc", "log", "openssl", - "openssl-probe", + "openssl-probe 0.1.6", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -1667,7 +2027,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -1716,12 +2076,75 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1762,7 +2185,7 @@ version = "0.10.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.13.1", "cfg-if", "foreign-types", "libc", @@ -1788,6 +2211,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "openssl-src" version = "300.4.2+3.4.1" @@ -1833,6 +2262,31 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax 0.8.5", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax 0.8.5", + "structmeta", + "syn 2.0.100", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -2112,7 +2566,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72c71c0c79b9701efe4e1e4b563b2016dd4ee789eb99badcb09d61ac4b92e4a2" dependencies = [ "libc", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -2149,8 +2603,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", ] [[package]] @@ -2160,7 +2624,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -2172,6 +2646,15 @@ dependencies = [ "getrandom 0.2.15", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.2", +] + [[package]] name = "rand_distr" version = "0.4.3" @@ -2179,7 +2662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand", + "rand 0.8.5", ] [[package]] @@ -2188,13 +2671,22 @@ version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.13.1", ] [[package]] @@ -2252,6 +2744,12 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.11.27" @@ -2289,7 +2787,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", + "webpki-roots 0.25.4", "winreg", ] @@ -2367,6 +2865,36 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rstest" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a2c585be59b6b5dd66a9d2084aa1d8bd52fbdb806eafdeffb52791147862035" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "825ea780781b15345a146be27eaefb05085e337e869bff01b4306a4fd4a9ad5a" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.100", + "unicode-ident", +] + [[package]] name = "rust-argon2" version = "0.8.3" @@ -2385,13 +2913,22 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -2404,7 +2941,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys 0.9.3", @@ -2441,13 +2978,27 @@ version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ + "log", "once_cell", + "ring 0.17.14", "rustls-pki-types", "rustls-webpki 0.103.1", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework 3.7.0", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -2537,6 +3088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25996b82292a7a57ed3508f052cfff8640d38d32018784acd714758b43da9c8f" dependencies = [ "bitcoin_hashes 0.12.0", + "rand 0.8.5", "secp256k1-sys", "serde", ] @@ -2556,8 +3108,21 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.9.0", - "core-foundation", + "bitflags 2.13.1", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.1", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -2565,44 +3130,61 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.14.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", ] +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" -version = "1.0.219" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.140" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", - "ryu", "serde", + "serde_core", + "zmij", ] [[package]] @@ -2614,6 +3196,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2626,6 +3219,36 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.8.0", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2670,7 +3293,7 @@ dependencies = [ "hex", "log", "openssl", - "rand", + "rand 0.8.5", "serde", "serde_json", "simln-lib", @@ -2699,15 +3322,15 @@ dependencies = [ "mockall", "mpsc", "ntest", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "rand_distr", "reqwest 0.12.15", "serde", "serde_json", "serde_millis", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-util", "tonic 0.8.3", @@ -2778,6 +3401,29 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.100", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "subtle" version = "2.6.1" @@ -2806,6 +3452,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "0.1.2" @@ -2839,7 +3496,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys 0.5.0", ] @@ -2849,8 +3506,8 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 2.9.0", - "core-foundation", + "bitflags 2.13.1", + "core-foundation 0.9.4", "system-configuration-sys 0.6.0", ] @@ -2893,13 +3550,52 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "testcontainers" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b92bce247dc9260a19808321e11b51ea6a0293d02b48ab1c6578960610cfa2a7" +dependencies = [ + "async-trait", + "bollard", + "bollard-stubs", + "bytes", + "docker_credential", + "either", + "etcetera", + "futures", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.19", + "tokio", + "tokio-stream", + "tokio-tar", + "tokio-util", + "ulid", + "url", +] + [[package]] name = "thiserror" version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl 2.0.19", ] [[package]] @@ -2913,6 +3609,17 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "thread_local" version = "1.1.8" @@ -3059,6 +3766,21 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "tokio-util" version = "0.7.14" @@ -3185,6 +3907,35 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "axum 0.8.9", + "base64 0.22.1", + "bytes", + "h2 0.4.8", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.6.0", + "hyper-timeout 0.5.2", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.5", + "socket2", + "tokio", + "tokio-stream", + "tower 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic-build" version = "0.8.4" @@ -3222,7 +3973,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", + "rand 0.8.5", "slab", "tokio", "tokio-util", @@ -3239,11 +3990,15 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", + "indexmap 2.8.0", "pin-project-lite", + "slab", "sync_wrapper 1.0.2", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -3328,6 +4083,16 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ulid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +dependencies = [ + "rand 0.9.5", + "web-time", +] + [[package]] name = "unicase" version = "2.8.1" @@ -3358,6 +4123,21 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "log", + "once_cell", + "rustls 0.23.25", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "url" version = "2.5.4" @@ -3367,6 +4147,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -3510,6 +4291,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki" version = "0.22.4" @@ -3526,6 +4317,24 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.9", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "4.4.2" @@ -3560,20 +4369,61 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "windows-link" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-registry" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ - "windows-result", - "windows-strings", + "windows-result 0.3.2", + "windows-strings 0.3.1", "windows-targets 0.53.0", ] @@ -3583,7 +4433,16 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" dependencies = [ - "windows-link", + "windows-link 0.1.1", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", ] [[package]] @@ -3592,7 +4451,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" dependencies = [ - "windows-link", + "windows-link 0.1.1", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", ] [[package]] @@ -3832,7 +4700,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.13.1", ] [[package]] @@ -3847,6 +4715,16 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.0.3", +] + [[package]] name = "yoke" version = "0.7.5" @@ -3959,3 +4837,9 @@ dependencies = [ "quote", "syn 2.0.100", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index fd5e2e96..2a13c1a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,4 +4,5 @@ resolver = "2" members = [ "sim-cli", "simln-lib", + "integration-tests", ] diff --git a/Makefile b/Makefile index 8feef81f..bdb27061 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,8 @@ help: @echo " run-docker Runs the Docker container in detached mode." @echo " run-interactive Runs the Docker container in interactive mode." @echo " stop-docker Stops the Docker container." + @echo " integration-sim Runs integration tests against a simulated network (no docker needed)." + @echo " integration-real Runs integration tests against real nodes in docker containers." @echo " check Runs code formatting and linting checks." @echo " check-code Runs code formatting and linting without stability check." @echo " format Fixes both formatting and linting issues in one go." @@ -41,6 +43,13 @@ run-interactive: stop-docker: docker stop sim-ln +integration-sim: + cargo test -p integration-tests --test sim_matrix + +integration-real: + @docker info > /dev/null 2>&1 || { echo "Error: no reachable docker daemon; integration-real needs a running docker runtime (Docker Desktop, colima or OrbStack)."; exit 1; } + cargo test -p integration-tests --test real_nodes -- --ignored --nocapture + check-code: $(FMT_CMD) --check $(CLIPPY_CMD) -- -D warnings diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml new file mode 100644 index 00000000..af3812fd --- /dev/null +++ b/integration-tests/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "integration-tests" +version = "0.1.0" +edition = "2021" +license = "MIT" +publish = false +description = """ +Integration test framework for sim-ln, covering simulated and real (dockerized) lightning networks. +""" + +[dependencies] +anyhow = { version = "1.0.69", features = ["backtrace"] } +bitcoin = { version = "0.30.1", features = ["serde"] } +bitcoincore-rpc = "0.17" +cln-grpc = "0.1.3" +csv = "1.2.2" +hex = "0.4.3" +ldk-server-client = { git = "https://github.com/lightningdevkit/ldk-server", rev = "8163f4fe139368613959bf4f10b19ee6a5b9b4ab" } +log = "0.4.20" +reqwest = { version = "0.12", features = ["json", "multipart"] } +serde_json = "1.0.104" +simln-lib = { path = "../simln-lib", features = ["virtual-time"] } +sim-cli = { path = "../sim-cli" } +# Pinned to the last version whose MSRV is compatible with the repo's Rust 1.85 toolchain. +testcontainers = "=0.25.0" +tokio = { version = "1.31.0", features = ["full"] } +tokio-util = { version = "0.7.13", features = ["rt"] } +tonic = { version = "0.8", features = ["tls", "transport"] } +tonic_lnd = { package = "fedimint-tonic-lnd", version = "0.1.2", features = ["lightningrpc"] } + +[dev-dependencies] +ntest = "0.9.0" +rstest = "0.23.0" +tempfile = "3" diff --git a/integration-tests/README.md b/integration-tests/README.md new file mode 100644 index 00000000..5eba814f --- /dev/null +++ b/integration-tests/README.md @@ -0,0 +1,46 @@ +# Integration Tests + +An integration test framework for sim-ln, split into two tiers: + +- **Simulated tier** (`tests/sim_matrix.rs`): runs sim-ln against in-process simulated + networks on virtual time. Covers the config-file surface (alias vs pubkey references, + scalar vs range values, connector inference), payment modalities (defined and random) + and negative validation cases. No external dependencies; completes in seconds. +- **Real-node tier** (`tests/real_nodes.rs`): spins up a heterogeneous regtest network in + docker — bitcoind plus one node each of LND, CLN, Eclair and ldk-server, connected in a + ring of channels — and runs simulations against it through each connector. Marked + `#[ignore]` so it only runs when asked for explicitly. + +## Running + +```sh +make integration-sim # simulated tier, no docker required +make integration-real # real-node tier, requires a docker runtime +``` + +## Prerequisites for the real-node tier + +- A running docker daemon. On macOS, Docker Desktop, colima and OrbStack all work; the + test harness discovers the socket automatically via testcontainers. +- `protoc` (already required to build the workspace). +- Network access to pull the pinned node images on first run. + +ldk-server has no published image, so on first use the harness runs `docker build` +against the upstream repository at the same rev the workspace's `ldk-server-client` +dependency pins, using upstream's own Dockerfile. The build takes a few minutes once and +is cached by docker thereafter. Set `SIMLN_LDK_SERVER_IMAGE=` to use a +prebuilt image instead — CI does this to reuse a cached build. + +## Layout + +- `src/env/` — the environment layer: provisions a network (simulated, or containers) and + emits its part of the sim.json config. Knows nothing about payments. +- `src/scenario.rs` — the scenario layer: describes activity (defined/random) and config + style. Knows nothing about how nodes are provisioned. +- `src/runner.rs` — assembles sim.json files, runs them through the same public entry + points the sim-cli binary uses, and collects results. +- `src/asserts.rs` — shared assertions over simulation output. +- `src/retry.rs` — capped-exponential-backoff polling used throughout real-node startup. + +The layering is the point: adding a node implementation touches only `src/env/containers/`, +and adding a payment scenario touches only the scenario layer and tests. diff --git a/integration-tests/src/asserts.rs b/integration-tests/src/asserts.rs new file mode 100644 index 00000000..3aad27b5 --- /dev/null +++ b/integration-tests/src/asserts.rs @@ -0,0 +1,166 @@ +//! Shared assertions over simulation output, independent of which environment produced it. + +use bitcoin::secp256k1::PublicKey; +use std::collections::HashMap; + +use crate::env::NodeHandle; +use crate::runner::SimOutput; +use crate::scenario::ValueSpec; + +/// Asserts that the validated activities resolved to the expected source/destination handles, in +/// order. This checks alias/pubkey resolution end to end: however the config referenced the node, +/// validation must land on the right pubkey. +pub fn assert_activities_resolved(out: &SimOutput, expected: &[(&NodeHandle, &NodeHandle)]) { + assert_eq!( + out.activities.len(), + expected.len(), + "expected {} validated activities, got {}", + expected.len(), + out.activities.len() + ); + + for (activity, (source, dest)) in out.activities.iter().zip(expected) { + assert_eq!( + activity.source.pubkey, source.pubkey, + "activity source resolved to {} instead of {} ({})", + activity.source.pubkey, source.pubkey, source.alias + ); + assert_eq!( + activity.destination.pubkey, dest.pubkey, + "activity destination resolved to {} instead of {} ({})", + activity.destination.pubkey, dest.pubkey, dest.alias + ); + } +} + +/// Asserts that exactly `expected` payments were dispatched, and that the results CSV recorded +/// every one of them. +pub fn assert_total_payments(out: &SimOutput, expected: u64) { + assert_eq!( + out.total_payments, expected, + "expected {expected} payments, simulation reported {}", + out.total_payments + ); + assert_eq!( + out.records.len() as u64, + expected, + "expected {expected} payment records in results CSV, found {}", + out.records.len() + ); +} + +/// Asserts that at least one payment was dispatched and that the CSV agrees with the simulation's +/// own count. Used for random activity, where exact counts depend on the generator. +pub fn assert_payments_dispatched(out: &SimOutput) { + assert!( + out.total_payments > 0, + "expected the simulation to dispatch payments, got none" + ); + assert_eq!( + out.records.len() as u64, + out.total_payments, + "results CSV has {} records but simulation reported {} payments", + out.records.len(), + out.total_payments + ); +} + +/// Asserts that every recorded payment succeeded. +pub fn assert_all_success(out: &SimOutput) { + let failures: Vec<_> = out.records.iter().filter(|r| !r.is_success()).collect(); + assert!( + failures.is_empty(), + "expected all payments to succeed, {} failed: {failures:?} (success rate {:.2}%)", + failures.len(), + out.success_rate + ); +} + +/// Asserts that every recorded payment amount could have been produced by the given spec. +pub fn assert_amounts_within(out: &SimOutput, spec: ValueSpec) { + for record in &out.records { + assert!( + spec.contains(record.amount_msat), + "payment of {} msat outside configured amount {spec:?}", + record.amount_msat + ); + } +} + +/// Asserts the results of count-bounded defined activity: payments flow only between the +/// configured pairs, every recorded payment succeeded, and each pair recorded either `count` or +/// `count - 1` payments. +/// +/// The final payment of a run is allowed to be missing because meeting a payment count shuts the +/// simulation down in the same instant as the last dispatch, and the results consumer prefers the +/// shutdown signal over draining pending results — so the last payment's record can be dropped. +pub fn assert_defined_payments(out: &SimOutput, pairs: &[(&NodeHandle, &NodeHandle)], count: u64) { + assert_all_success(out); + let counts = assert_payments_between(out, pairs); + + for (source, dest) in pairs { + let recorded = counts + .get(&(source.pubkey, dest.pubkey)) + .copied() + .unwrap_or(0); + assert!( + (count - 1..=count).contains(&recorded), + "expected {count} (or {} with the trailing record lost to shutdown) payments from {} \ + to {}, recorded {recorded}", + count - 1, + source.alias, + dest.alias + ); + } +} + +/// Asserts that recorded payments flow only between the expected (source, destination) pairs, and +/// returns the per-pair counts for further assertions. +pub fn assert_payments_between( + out: &SimOutput, + pairs: &[(&NodeHandle, &NodeHandle)], +) -> HashMap<(PublicKey, PublicKey), u64> { + let allowed: Vec<(PublicKey, PublicKey)> = + pairs.iter().map(|(s, d)| (s.pubkey, d.pubkey)).collect(); + let mut counts: HashMap<(PublicKey, PublicKey), u64> = HashMap::new(); + + for record in &out.records { + let pair = (record.source, record.destination); + assert!( + allowed.contains(&pair), + "payment from {} to {} not part of any configured activity", + record.source, + record.destination + ); + *counts.entry(pair).or_default() += 1; + } + + counts +} + +/// Asserts that no recorded payment involves any of the given nodes, as sender or receiver. +pub fn assert_not_involved(out: &SimOutput, excluded: &[&NodeHandle]) { + for record in &out.records { + for node in excluded { + assert!( + record.source != node.pubkey && record.destination != node.pubkey, + "excluded node {} ({}) took part in payment {} -> {}", + node.alias, + node.pubkey, + record.source, + record.destination + ); + } + } +} + +/// Asserts that every payment source is one of the given controlled nodes. +pub fn assert_sources_controlled(out: &SimOutput, controlled: &[NodeHandle]) { + for record in &out.records { + assert!( + controlled.iter().any(|n| n.pubkey == record.source), + "payment source {} is not a controlled node", + record.source + ); + } +} diff --git a/integration-tests/src/env/containers/bitcoind.rs b/integration-tests/src/env/containers/bitcoind.rs new file mode 100644 index 00000000..f021a24a --- /dev/null +++ b/integration-tests/src/env/containers/bitcoind.rs @@ -0,0 +1,142 @@ +//! The bitcoind container backing the regtest network: chain source for every node, miner, and +//! on-chain wallet (which Eclair also spends from directly, since it has no wallet of its own). + +use std::str::FromStr; +use std::time::Duration; + +use anyhow::Context; +use bitcoin::Address; +use bitcoincore_rpc::{Auth, Client, RpcApi}; +use testcontainers::core::IntoContainerPort; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, GenericImage, ImageExt}; + +use super::{dump_logs, BTC_RPC_PASS, BTC_RPC_USER}; +use crate::retry::{with_backoff, Backoff}; + +/// Multi-arch maintainer-published Bitcoin Core image. Eclair 0.14.x requires Core 31+. +const IMAGE: &str = "bitcoin/bitcoin"; +const TAG: &str = "31.1"; + +/// Regtest RPC port inside the container. +const RPC_PORT: u16 = 18443; + +/// ZMQ endpoints: LND consumes rawblock + rawtx, Eclair consumes hashblock + rawtx. +pub const ZMQ_RAWBLOCK_PORT: u16 = 28332; +pub const ZMQ_RAWTX_PORT: u16 = 28333; +pub const ZMQ_HASHBLOCK_PORT: u16 = 28334; + +const WALLET: &str = "simln"; + +pub struct Bitcoind { + container: ContainerAsync, + name: String, + wallet_rpc: Client, +} + +impl Bitcoind { + pub async fn start(docker_network: &str) -> anyhow::Result { + let name = format!("simln-bitcoind-{}", std::process::id()); + let container = GenericImage::new(IMAGE, TAG) + .with_exposed_port(RPC_PORT.tcp()) + .with_network(docker_network) + .with_container_name(&name) + .with_startup_timeout(Duration::from_secs(600)) + .with_cmd([ + "-regtest", + "-server=1", + "-txindex=1", + "-rpcbind=0.0.0.0", + "-rpcallowip=0.0.0.0/0", + &format!("-rpcuser={BTC_RPC_USER}"), + &format!("-rpcpassword={BTC_RPC_PASS}"), + &format!("-zmqpubrawblock=tcp://0.0.0.0:{ZMQ_RAWBLOCK_PORT}"), + &format!("-zmqpubrawtx=tcp://0.0.0.0:{ZMQ_RAWTX_PORT}"), + &format!("-zmqpubhashblock=tcp://0.0.0.0:{ZMQ_HASHBLOCK_PORT}"), + "-fallbackfee=0.0002", + "-addresstype=bech32m", + "-changetype=bech32m", + ]) + .start() + .await + .context("starting bitcoind container")?; + + let host_port = container.get_host_port_ipv4(RPC_PORT).await?; + let auth = Auth::UserPass(BTC_RPC_USER.to_string(), BTC_RPC_PASS.to_string()); + let base_rpc = Client::new(&format!("http://127.0.0.1:{host_port}"), auth.clone())?; + + let setup = async { + with_backoff("bitcoind rpc ready", Backoff::default(), || async { + // Untyped call: the crate's typed getblockchaininfo struct predates Core 31's + // response format (warnings became an array). + base_rpc.call::("getblockchaininfo", &[]) + }) + .await?; + + base_rpc + .create_wallet(WALLET, None, None, None, None) + .context("creating bitcoind wallet")?; + + let wallet_rpc = Client::new( + &format!("http://127.0.0.1:{host_port}/wallet/{WALLET}"), + auth, + )?; + + // Mine past coinbase maturity so the wallet has spendable funds. + let address = new_address(&wallet_rpc)?; + wallet_rpc.generate_to_address(101, &address)?; + anyhow::Ok(wallet_rpc) + }; + + match setup.await { + Ok(wallet_rpc) => Ok(Bitcoind { + container, + name, + wallet_rpc, + }), + Err(e) => { + dump_logs("bitcoind", &container).await; + Err(e) + }, + } + } + + pub fn container_name(&self) -> &str { + &self.name + } + + pub fn container(&self) -> &ContainerAsync { + &self.container + } + + /// Mines `blocks` blocks to the harness wallet. + pub fn mine(&self, blocks: u64) -> anyhow::Result<()> { + let address = new_address(&self.wallet_rpc)?; + self.wallet_rpc.generate_to_address(blocks, &address)?; + Ok(()) + } + + /// Sends `amount_sat` to each of the given addresses and mines a block to confirm. + pub fn fund(&self, addresses: &[String], amount_sat: u64) -> anyhow::Result<()> { + for addr in addresses { + let addr = Address::from_str(addr) + .with_context(|| format!("parsing funding address {addr}"))? + .assume_checked(); + self.wallet_rpc.send_to_address( + &addr, + bitcoin::Amount::from_sat(amount_sat), + None, + None, + None, + None, + None, + None, + )?; + } + self.mine(1) + } +} + +fn new_address(rpc: &Client) -> anyhow::Result
{ + Ok(rpc.get_new_address(None, None)?.assume_checked()) +} diff --git a/integration-tests/src/env/containers/cln.rs b/integration-tests/src/env/containers/cln.rs new file mode 100644 index 00000000..ce0f0bc9 --- /dev/null +++ b/integration-tests/src/env/containers/cln.rs @@ -0,0 +1,287 @@ +//! Core Lightning running in the official elementsproject image, driven over its mTLS gRPC +//! interface from the host. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{anyhow, Context}; +use bitcoin::secp256k1::PublicKey; +use cln_grpc::pb; +use cln_grpc::pb::node_client::NodeClient; +use serde_json::{json, Value}; +use testcontainers::core::IntoContainerPort; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, GenericImage, ImageExt}; +use tonic::transport::{Certificate, Channel, ClientTlsConfig, Identity}; + +use super::{dump_logs, extract_file, BTC_RPC_PASS, BTC_RPC_USER, CHANNEL_SIZE_SAT, P2P_PORT}; +use crate::env::{NodeHandle, NodeImpl}; +use crate::retry::{with_backoff, Backoff}; + +const IMAGE: &str = "elementsproject/lightningd"; +const TAG: &str = "v26.06.6"; + +const GRPC_PORT: u16 = 9736; +const ALIAS: &str = "cln"; + +pub struct ClnHarness { + container: ContainerAsync, + name: String, + pub pubkey: PublicKey, + channel: Channel, + grpc_host_port: u16, + ca_path: PathBuf, + client_cert_path: PathBuf, + client_key_path: PathBuf, +} + +impl ClnHarness { + pub async fn start( + docker_network: &str, + bitcoind_name: &str, + creds_dir: &Path, + ) -> anyhow::Result { + let name = format!("simln-cln-{}", std::process::id()); + let container = GenericImage::new(IMAGE, TAG) + .with_exposed_port(GRPC_PORT.tcp()) + .with_network(docker_network) + .with_container_name(&name) + .with_startup_timeout(Duration::from_secs(600)) + .with_env_var("LIGHTNINGD_NETWORK", "regtest") + .with_cmd([ + format!("--bitcoin-rpcconnect={bitcoind_name}"), + "--bitcoin-rpcport=18443".to_string(), + format!("--bitcoin-rpcuser={BTC_RPC_USER}"), + format!("--bitcoin-rpcpassword={BTC_RPC_PASS}"), + format!("--bind-addr=0.0.0.0:{P2P_PORT}"), + "--grpc-host=0.0.0.0".to_string(), + format!("--grpc-port={GRPC_PORT}"), + format!("--alias={ALIAS}"), + // Zero fees so multi-hop payments through this node never hit fee budgets. + "--fee-base=0".to_string(), + "--fee-per-satoshi=0".to_string(), + ]) + .start() + .await + .context("starting cln container")?; + + let init = async { + let ca_path = creds_dir.join("cln/ca.pem"); + let client_cert_path = creds_dir.join("cln/client.pem"); + let client_key_path = creds_dir.join("cln/client-key.pem"); + + let ca = extract_file( + &container, + "/root/.lightning/regtest/ca.pem", + &ca_path, + Backoff::slow(), + ) + .await?; + let cert = extract_file( + &container, + "/root/.lightning/regtest/client.pem", + &client_cert_path, + Backoff::slow(), + ) + .await?; + let key = extract_file( + &container, + "/root/.lightning/regtest/client-key.pem", + &client_key_path, + Backoff::slow(), + ) + .await?; + + let grpc_host_port = container.get_host_port_ipv4(GRPC_PORT).await?; + // The server cert's SANs are "cln" and "localhost"; override the domain the same way + // sim-ln's connector does since we dial by IP. + let tls = ClientTlsConfig::new() + .domain_name("cln") + .ca_certificate(Certificate::from_pem(&ca)) + .identity(Identity::from_pem(&cert, &key)); + + let channel = with_backoff("cln grpc connect", Backoff::slow(), || async { + Channel::from_shared(format!("https://127.0.0.1:{grpc_host_port}"))? + .tls_config(tls.clone())? + .connect() + .await + .map_err(anyhow::Error::from) + }) + .await?; + + let info = with_backoff("cln synced", Backoff::slow(), || async { + let info = NodeClient::new(channel.clone()) + .getinfo(pb::GetinfoRequest {}) + .await + .map_err(|e| anyhow!(e.to_string()))? + .into_inner(); + if info.warning_bitcoind_sync.is_none() && info.warning_lightningd_sync.is_none() { + Ok(info) + } else { + Err(anyhow!("still syncing with bitcoind")) + } + }) + .await?; + + let pubkey = PublicKey::from_slice(&info.id).context("parsing cln pubkey")?; + anyhow::Ok(( + channel, + pubkey, + grpc_host_port, + ca_path, + client_cert_path, + client_key_path, + )) + }; + + match init.await { + Ok((channel, pubkey, grpc_host_port, ca_path, client_cert_path, client_key_path)) => { + Ok(ClnHarness { + container, + name, + pubkey, + channel, + grpc_host_port, + ca_path, + client_cert_path, + client_key_path, + }) + }, + Err(e) => { + dump_logs("cln", &container).await; + Err(e.context("cln startup")) + }, + } + } + + fn client(&self) -> NodeClient { + NodeClient::new(self.channel.clone()) + } + + pub fn container(&self) -> &ContainerAsync { + &self.container + } + + pub fn p2p_address(&self) -> String { + format!("{}:{P2P_PORT}", self.name) + } + + pub fn node_handle(&self) -> NodeHandle { + NodeHandle { + pubkey: self.pubkey, + alias: ALIAS.to_string(), + implementation: NodeImpl::Cln, + } + } + + pub fn sim_config(&self) -> Value { + json!({ + "id": self.pubkey.to_string(), + "address": format!("https://127.0.0.1:{}", self.grpc_host_port), + "ca_cert": self.ca_path.to_string_lossy(), + "client_cert": self.client_cert_path.to_string_lossy(), + "client_key": self.client_key_path.to_string_lossy(), + }) + } + + pub async fn new_address(&self) -> anyhow::Result { + let response = self + .client() + .new_addr(pb::NewaddrRequest::default()) + .await? + .into_inner(); + response + .bech32 + .ok_or_else(|| anyhow!("cln newaddr returned no bech32 address")) + } + + pub async fn open_channel( + &self, + peer: &PublicKey, + peer_host: &str, + peer_port: u16, + ) -> anyhow::Result<()> { + with_backoff("cln open channel", Backoff::slow(), || async { + let _ = self + .client() + .connect_peer(pb::ConnectRequest { + id: peer.to_string(), + host: Some(peer_host.to_string()), + port: Some(peer_port as u32), + }) + .await; + + self.client() + .fund_channel(pb::FundchannelRequest { + id: peer.serialize().to_vec(), + amount: Some(pb::AmountOrAll { + value: Some(pb::amount_or_all::Value::Amount(pb::Amount { + msat: CHANNEL_SIZE_SAT * 1000, + })), + }), + announce: Some(true), + ..Default::default() + }) + .await + .map_err(|e| anyhow!(e.to_string())) + }) + .await?; + Ok(()) + } + + pub async fn active_channel_count(&self) -> anyhow::Result { + let response = self + .client() + .list_peer_channels(pb::ListpeerchannelsRequest::default()) + .await? + .into_inner(); + let normal = + pb::listpeerchannels_channels::ListpeerchannelsChannelsState::ChanneldNormal as i32; + Ok(response + .channels + .iter() + .filter(|c| c.state == normal) + .count()) + } + + pub async fn graph_synced(&self, nodes: usize, channels: usize) -> anyhow::Result<()> { + let node_count = self + .client() + .list_nodes(pb::ListnodesRequest::default()) + .await? + .into_inner() + .nodes + .len(); + // CLN's gossip view lists one entry per direction. + let direction_count = self + .client() + .list_channels(pb::ListchannelsRequest::default()) + .await? + .into_inner() + .channels + .len(); + + if node_count >= nodes && direction_count >= channels * 2 { + Ok(()) + } else { + Err(anyhow!( + "graph has {node_count}/{nodes} nodes, {direction_count}/{} channel directions", + channels * 2 + )) + } + } + + pub async fn settled_keysend_count(&self) -> anyhow::Result { + let response = self + .client() + .list_invoices(pb::ListinvoicesRequest::default()) + .await? + .into_inner(); + let paid = pb::listinvoices_invoices::ListinvoicesInvoicesStatus::Paid as i32; + Ok(response + .invoices + .iter() + .filter(|i| i.status == paid && i.label.starts_with("keysend-")) + .count() as u64) + } +} diff --git a/integration-tests/src/env/containers/eclair.rs b/integration-tests/src/env/containers/eclair.rs new file mode 100644 index 00000000..d6f18778 --- /dev/null +++ b/integration-tests/src/env/containers/eclair.rs @@ -0,0 +1,249 @@ +//! Eclair running in ACINQ's image, driven over its REST API from the host. +//! +//! Two quirks relative to the other nodes: the image is amd64-only (it runs under emulation on +//! Apple Silicon, so it gets generous timeouts) and only ships a moving `latest` tag, so it is +//! pinned by digest. Configuration goes through `JAVA_OPTS` because the image's shell-form +//! entrypoint ignores container args. Eclair has no on-chain wallet of its own — it spends +//! directly from bitcoind's wallet, so it needs no separate funding. + +use std::collections::HashMap; +use std::time::Duration; + +use anyhow::{anyhow, Context}; +use bitcoin::secp256k1::PublicKey; +use serde_json::{json, Value}; +use testcontainers::core::IntoContainerPort; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, GenericImage, ImageExt}; + +use super::bitcoind::{ZMQ_HASHBLOCK_PORT, ZMQ_RAWTX_PORT}; +use super::{dump_logs, BTC_RPC_PASS, BTC_RPC_USER, CHANNEL_SIZE_SAT, P2P_PORT}; +use crate::env::{NodeHandle, NodeImpl}; +use crate::retry::{with_backoff, Backoff}; + +const IMAGE: &str = "acinq/eclair"; +// Versioned tags stopped at 0.8.0; `latest` is the maintained tag, pinned here by digest +// (0.14.1 at the time of pinning). +const TAG: &str = "latest@sha256:6eb7d528bc150822231d7d73cc6f27942d02c1682aa0e22c7a2b0b2dc3249aa3"; + +const API_PORT: u16 = 8080; +const API_PASSWORD: &str = "simln-api"; +const ALIAS: &str = "eclair"; + +pub struct EclairHarness { + container: ContainerAsync, + name: String, + pub pubkey: PublicKey, + http: reqwest::Client, + api_base: String, + api_host_port: u16, +} + +impl EclairHarness { + pub async fn start(docker_network: &str, bitcoind_name: &str) -> anyhow::Result { + let name = format!("simln-eclair-{}", std::process::id()); + let java_opts = [ + // ACINQ's image is built from the post-release "Back to dev" commit, which carries + // an unconditional guard against running dev builds. The guard checks nothing + // dynamic — this opt-out is required to start the image at all. + "-Declair.allow-unsafe-startup=true".to_string(), + "-Declair.chain=regtest".to_string(), + format!("-Declair.node-alias={ALIAS}"), + format!("-Declair.server.port={P2P_PORT}"), + "-Declair.api.enabled=true".to_string(), + "-Declair.api.binding-ip=0.0.0.0".to_string(), + format!("-Declair.api.port={API_PORT}"), + format!("-Declair.api.password={API_PASSWORD}"), + format!("-Declair.bitcoind.host={bitcoind_name}"), + "-Declair.bitcoind.rpcport=18443".to_string(), + format!("-Declair.bitcoind.rpcuser={BTC_RPC_USER}"), + format!("-Declair.bitcoind.rpcpassword={BTC_RPC_PASS}"), + "-Declair.bitcoind.wallet=simln".to_string(), + // Eclair's zmqblock endpoint expects hashblock, not rawblock. + format!("-Declair.bitcoind.zmqblock=tcp://{bitcoind_name}:{ZMQ_HASHBLOCK_PORT}"), + format!("-Declair.bitcoind.zmqtx=tcp://{bitcoind_name}:{ZMQ_RAWTX_PORT}"), + "-Declair.features.keysend=optional".to_string(), + // CLN -> Eclair keysend interop needs a min final expiry of at least CLN's 22-block + // default, with the fulfill safety margin strictly below it. + "-Declair.channel.min-final-expiry-delta-blocks=24".to_string(), + "-Declair.channel.fulfill-safety-before-timeout-blocks=12".to_string(), + // Zero fees so multi-hop payments through this node never hit fee budgets. + "-Declair.relay.fees.public-channels.fee-base-msat=0".to_string(), + "-Declair.relay.fees.public-channels.fee-proportional-millionths=0".to_string(), + ] + .join(" "); + + let container = GenericImage::new(IMAGE, TAG) + .with_exposed_port(API_PORT.tcp()) + .with_network(docker_network) + .with_container_name(&name) + .with_startup_timeout(Duration::from_secs(600)) + .with_env_var("JAVA_OPTS", java_opts) + .start() + .await + .context("starting eclair container")?; + + let init = async { + let api_host_port = container.get_host_port_ipv4(API_PORT).await?; + let api_base = format!("http://127.0.0.1:{api_host_port}"); + let http = reqwest::Client::new(); + + // The JVM (under emulation on Apple Silicon) takes a while; poll until the API + // answers getinfo. + let info = with_backoff("eclair api ready", Backoff::slow(), || async { + api_call(&http, &api_base, "getinfo", &[]).await + }) + .await?; + + let pubkey = info + .get("nodeId") + .and_then(Value::as_str) + .ok_or_else(|| anyhow!("getinfo response missing nodeId: {info}"))? + .parse() + .context("parsing eclair pubkey")?; + anyhow::Ok((http, api_base, api_host_port, pubkey)) + }; + + match init.await { + Ok((http, api_base, api_host_port, pubkey)) => Ok(EclairHarness { + container, + name, + pubkey, + http, + api_base, + api_host_port, + }), + Err(e) => { + dump_logs("eclair", &container).await; + Err(e.context("eclair startup")) + }, + } + } + + async fn api(&self, endpoint: &str, params: &[(&str, String)]) -> anyhow::Result { + api_call(&self.http, &self.api_base, endpoint, params).await + } + + pub fn container(&self) -> &ContainerAsync { + &self.container + } + + pub fn container_name(&self) -> &str { + &self.name + } + + pub fn node_handle(&self) -> NodeHandle { + NodeHandle { + pubkey: self.pubkey, + alias: ALIAS.to_string(), + implementation: NodeImpl::Eclair, + } + } + + pub fn sim_config(&self) -> Value { + json!({ + "id": self.pubkey.to_string(), + "base_url": format!("http://127.0.0.1:{}", self.api_host_port), + "api_username": "", + "api_password": API_PASSWORD, + }) + } + + /// Connects and opens an announced channel, retrying until bitcoind's wallet (which eclair + /// spends from) can fund it. + pub async fn open_channel(&self, peer: &PublicKey, peer_addr: &str) -> anyhow::Result<()> { + with_backoff("eclair open channel", Backoff::slow(), || async { + let _ = self + .api("connect", &[("uri", format!("{peer}@{peer_addr}"))]) + .await; + + self.api( + "open", + &[ + ("nodeId", peer.to_string()), + ("fundingSatoshis", CHANNEL_SIZE_SAT.to_string()), + ("announceChannel", "true".to_string()), + ], + ) + .await + }) + .await?; + Ok(()) + } + + pub async fn active_channel_count(&self) -> anyhow::Result { + let channels = self.api("channels", &[]).await?; + let channels = channels + .as_array() + .ok_or_else(|| anyhow!("channels response is not an array: {channels}"))?; + Ok(channels + .iter() + .filter(|c| c.get("state").and_then(Value::as_str) == Some("NORMAL")) + .count()) + } + + pub async fn graph_synced(&self, nodes: usize, channels: usize) -> anyhow::Result<()> { + let node_count = self + .api("nodes", &[]) + .await? + .as_array() + .map(Vec::len) + .unwrap_or(0); + let channel_count = self + .api("allchannels", &[]) + .await? + .as_array() + .map(Vec::len) + .unwrap_or(0); + + if node_count >= nodes && channel_count >= channels { + Ok(()) + } else { + Err(anyhow!( + "graph has {node_count}/{nodes} nodes, {channel_count}/{channels} channels" + )) + } + } + + pub async fn settled_keysend_count(&self) -> anyhow::Result { + let payments = self.api("listreceivedpayments", &[]).await?; + let payments = payments + .as_array() + .ok_or_else(|| anyhow!("listreceivedpayments is not an array: {payments}"))?; + Ok(payments + .iter() + .filter(|p| { + p.get("paymentType").and_then(Value::as_str) == Some("KeySend") + && p.pointer("/status/type").and_then(Value::as_str) == Some("received") + }) + .count() as u64) + } +} + +async fn api_call( + http: &reqwest::Client, + base: &str, + endpoint: &str, + params: &[(&str, String)], +) -> anyhow::Result { + let mut form = HashMap::new(); + for (key, value) in params { + form.insert(*key, value.clone()); + } + + let response = http + .post(format!("{base}/{endpoint}")) + .basic_auth("", Some(API_PASSWORD)) + .form(&form) + .send() + .await + .with_context(|| format!("eclair {endpoint} request"))?; + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + if !status.is_success() { + return Err(anyhow!("eclair {endpoint} returned {status}: {body}")); + } + + serde_json::from_str(&body).with_context(|| format!("eclair {endpoint} response: {body}")) +} diff --git a/integration-tests/src/env/containers/ldk_server.rs b/integration-tests/src/env/containers/ldk_server.rs new file mode 100644 index 00000000..cfd53ead --- /dev/null +++ b/integration-tests/src/env/containers/ldk_server.rs @@ -0,0 +1,317 @@ +//! ldk-server built from the upstream repository at the rev this workspace's client dependency +//! pins, and driven over its authenticated gRPC API from the host. +//! +//! There is no published ldk-server image: the harness builds one with `docker build` against the +//! upstream git URL (using upstream's own Dockerfile) unless `SIMLN_LDK_SERVER_IMAGE` names a +//! prebuilt image — CI sets it to reuse a cached build. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{anyhow, Context}; +use bitcoin::secp256k1::PublicKey; +use ldk_server_client::client::LdkServerClient; +use ldk_server_client::ldk_server_grpc::api::{ + GetNodeInfoRequest, GraphListChannelsRequest, GraphListNodesRequest, ListChannelsRequest, + ListPaymentsRequest, OnchainReceiveRequest, OpenChannelRequest, +}; +use ldk_server_client::ldk_server_grpc::types::{payment_kind, PaymentDirection, PaymentStatus}; +use serde_json::{json, Value}; +use testcontainers::core::IntoContainerPort; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, GenericImage, ImageExt}; + +use super::{ + dump_logs, exec_stdout, extract_file, BTC_RPC_PASS, BTC_RPC_USER, CHANNEL_SIZE_SAT, P2P_PORT, +}; +use crate::env::{NodeHandle, NodeImpl}; +use crate::retry::{with_backoff, Backoff}; + +/// The rev the workspace's ldk-server-client dependency pins; the server is built from the same +/// rev so client and server always match. +const LDK_SERVER_REV: &str = "8163f4fe139368613959bf4f10b19ee6a5b9b4ab"; +const LDK_SERVER_REPO: &str = "https://github.com/lightningdevkit/ldk-server.git"; + +/// Env var naming a prebuilt ldk-server image to use instead of building one. +const IMAGE_ENV: &str = "SIMLN_LDK_SERVER_IMAGE"; + +const GRPC_PORT: u16 = 3536; +const ALIAS: &str = "ldk"; +const STORAGE_DIR: &str = "/data/ldk-server"; +const CONFIG_PATH: &str = "/config/ldk-server.toml"; + +pub struct LdkHarness { + container: ContainerAsync, + name: String, + pub pubkey: PublicKey, + client: LdkServerClient, + grpc_host_port: u16, + api_key_hex: String, + cert_path: PathBuf, +} + +impl LdkHarness { + pub async fn start( + docker_network: &str, + bitcoind_name: &str, + creds_dir: &Path, + ) -> anyhow::Result { + let (image, tag) = ensure_image().await?; + + let config = format!( + r#"[node] +network = "regtest" +listening_addresses = ["0.0.0.0:{P2P_PORT}"] +alias = "{ALIAS}" + +[storage.disk] +dir_path = "{STORAGE_DIR}" + +[log] +level = "Info" +log_to_file = false + +[bitcoind] +rpc_address = "{bitcoind_name}:18443" +rpc_user = "{BTC_RPC_USER}" +rpc_password = "{BTC_RPC_PASS}" +"# + ); + + let name = format!("simln-ldk-{}", std::process::id()); + let container = GenericImage::new(image, tag) + .with_exposed_port(GRPC_PORT.tcp()) + .with_network(docker_network) + .with_container_name(&name) + .with_startup_timeout(Duration::from_secs(600)) + .with_copy_to(CONFIG_PATH, config.into_bytes()) + .with_cmd([CONFIG_PATH]) + .start() + .await + .context("starting ldk-server container")?; + + let init = async { + // The api key and TLS cert are generated on first startup and are exactly what the + // client needs, so waiting for them doubles as a readiness check. + let cert_path = creds_dir.join("ldk/tls.crt"); + let cert_pem = extract_file( + &container, + &format!("{STORAGE_DIR}/tls.crt"), + &cert_path, + Backoff::slow(), + ) + .await?; + // The api key is 32 raw bytes on disk; clients use its lowercase hex encoding. + let api_key_hex = hex::encode( + with_backoff( + "waiting for ldk-server api key", + Backoff::slow(), + || async { + let bytes = exec_stdout( + &container, + &["cat", &format!("{STORAGE_DIR}/regtest/api_key")], + ) + .await?; + if bytes.is_empty() { + Err(anyhow!("api key file empty")) + } else { + Ok(bytes) + } + }, + ) + .await?, + ); + + let grpc_host_port = container.get_host_port_ipv4(GRPC_PORT).await?; + let client = LdkServerClient::new( + format!("127.0.0.1:{grpc_host_port}"), + api_key_hex.clone(), + &cert_pem, + ) + .map_err(|e| anyhow!("creating ldk-server client: {e}"))?; + + let info = with_backoff("ldk-server ready", Backoff::slow(), || async { + client.get_node_info(GetNodeInfoRequest {}).await + }) + .await?; + + let pubkey = info.node_id.parse().context("parsing ldk-server pubkey")?; + anyhow::Ok((client, pubkey, grpc_host_port, api_key_hex, cert_path)) + }; + + match init.await { + Ok((client, pubkey, grpc_host_port, api_key_hex, cert_path)) => Ok(LdkHarness { + container, + name, + pubkey, + client, + grpc_host_port, + api_key_hex, + cert_path, + }), + Err(e) => { + dump_logs("ldk-server", &container).await; + Err(e.context("ldk-server startup")) + }, + } + } + + pub fn container(&self) -> &ContainerAsync { + &self.container + } + + pub fn p2p_address(&self) -> String { + format!("{}:{P2P_PORT}", self.name) + } + + pub fn node_handle(&self) -> NodeHandle { + NodeHandle { + pubkey: self.pubkey, + alias: ALIAS.to_string(), + implementation: NodeImpl::LdkServer, + } + } + + pub fn sim_config(&self) -> Value { + json!({ + "address": format!("https://127.0.0.1:{}", self.grpc_host_port), + "api_key": self.api_key_hex, + "cert": self.cert_path.to_string_lossy(), + }) + } + + pub async fn new_address(&self) -> anyhow::Result { + let response = self + .client + .onchain_receive(OnchainReceiveRequest {}) + .await + .map_err(|e| anyhow!("ldk-server onchain_receive: {e}"))?; + Ok(response.address) + } + + pub async fn open_channel(&self, peer: &PublicKey, peer_addr: &str) -> anyhow::Result<()> { + with_backoff("ldk-server open channel", Backoff::slow(), || async { + self.client + .open_channel(OpenChannelRequest { + node_pubkey: peer.to_string(), + address: peer_addr.to_string(), + channel_amount_sats: CHANNEL_SIZE_SAT, + announce_channel: true, + ..Default::default() + }) + .await + }) + .await?; + Ok(()) + } + + pub async fn channel_count(&self) -> anyhow::Result { + let response = self + .client + .list_channels(ListChannelsRequest {}) + .await + .map_err(|e| anyhow!("ldk-server list_channels: {e}"))?; + Ok(response.channels.len()) + } + + pub async fn graph_synced(&self, nodes: usize, channels: usize) -> anyhow::Result<()> { + let node_count = self + .client + .graph_list_nodes(GraphListNodesRequest {}) + .await + .map_err(|e| anyhow!("ldk-server graph_list_nodes: {e}"))? + .node_ids + .len(); + let channel_count = self + .client + .graph_list_channels(GraphListChannelsRequest {}) + .await + .map_err(|e| anyhow!("ldk-server graph_list_channels: {e}"))? + .short_channel_ids + .len(); + + if node_count >= nodes && channel_count >= channels { + Ok(()) + } else { + Err(anyhow!( + "graph has {node_count}/{nodes} nodes, {channel_count}/{channels} channels" + )) + } + } + + pub async fn settled_keysend_count(&self) -> anyhow::Result { + let mut count = 0u64; + let mut page_token = None; + loop { + let response = self + .client + .list_payments(ListPaymentsRequest { page_token }) + .await + .map_err(|e| anyhow!("ldk-server list_payments: {e}"))?; + + count += response + .payments + .iter() + .filter(|p| { + p.direction == PaymentDirection::Inbound as i32 + && p.status == PaymentStatus::Succeeded as i32 + && matches!( + p.kind.as_ref().and_then(|k| k.kind.as_ref()), + Some(payment_kind::Kind::Spontaneous(_)) + ) + }) + .count() as u64; + + match response.next_page_token { + Some(token) => page_token = Some(token), + None => return Ok(count), + } + } + } +} + +/// Returns the (image, tag) to run, building the image from the pinned upstream rev if neither a +/// `SIMLN_LDK_SERVER_IMAGE` override nor a previously built image is available. +async fn ensure_image() -> anyhow::Result<(String, String)> { + if let Ok(image) = std::env::var(IMAGE_ENV) { + return split_image(&image); + } + + let tag = format!("simln-ldk-server:{}", &LDK_SERVER_REV[..12]); + let exists = tokio::process::Command::new("docker") + .args(["image", "inspect", &tag]) + .output() + .await + .context("checking for ldk-server image")? + .status + .success(); + + if !exists { + eprintln!("building ldk-server image from {LDK_SERVER_REPO}#{LDK_SERVER_REV} (one-time, takes a few minutes)..."); + let output = tokio::process::Command::new("docker") + .args([ + "build", + "-t", + &tag, + &format!("{LDK_SERVER_REPO}#{LDK_SERVER_REV}"), + ]) + .output() + .await + .context("running docker build for ldk-server")?; + if !output.status.success() { + return Err(anyhow!( + "docker build of ldk-server failed:\n{}", + String::from_utf8_lossy(&output.stderr) + )); + } + } + + split_image(&tag) +} + +fn split_image(image: &str) -> anyhow::Result<(String, String)> { + match image.rsplit_once(':') { + Some((name, tag)) => Ok((name.to_string(), tag.to_string())), + None => Ok((image.to_string(), "latest".to_string())), + } +} diff --git a/integration-tests/src/env/containers/lnd.rs b/integration-tests/src/env/containers/lnd.rs new file mode 100644 index 00000000..6e1ad2a5 --- /dev/null +++ b/integration-tests/src/env/containers/lnd.rs @@ -0,0 +1,258 @@ +//! LND running in the official lightninglabs image, driven over gRPC from the host. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{anyhow, Context}; +use bitcoin::secp256k1::PublicKey; +use serde_json::{json, Value}; +use testcontainers::core::IntoContainerPort; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, GenericImage, ImageExt}; +use tonic_lnd::lnrpc; + +use super::bitcoind::{ZMQ_RAWBLOCK_PORT, ZMQ_RAWTX_PORT}; +use super::{dump_logs, extract_file, BTC_RPC_PASS, BTC_RPC_USER, CHANNEL_SIZE_SAT, P2P_PORT}; +use crate::env::{NodeHandle, NodeImpl}; +use crate::retry::{with_backoff, Backoff}; + +const IMAGE: &str = "lightninglabs/lnd"; +const TAG: &str = "v0.21.1-beta"; + +const GRPC_PORT: u16 = 10009; +const ALIAS: &str = "lnd"; + +pub struct LndHarness { + container: ContainerAsync, + name: String, + pub pubkey: PublicKey, + client: tonic_lnd::Client, + grpc_host_port: u16, + cert_path: PathBuf, + macaroon_path: PathBuf, +} + +impl LndHarness { + pub async fn start( + docker_network: &str, + bitcoind_name: &str, + creds_dir: &Path, + ) -> anyhow::Result { + let name = format!("simln-lnd-{}", std::process::id()); + let container = GenericImage::new(IMAGE, TAG) + .with_exposed_port(GRPC_PORT.tcp()) + .with_network(docker_network) + .with_container_name(&name) + .with_startup_timeout(Duration::from_secs(600)) + .with_cmd([ + "--noseedbackup", + "--bitcoin.regtest", + "--bitcoin.node=bitcoind", + &format!("--bitcoind.rpchost={bitcoind_name}:18443"), + &format!("--bitcoind.rpcuser={BTC_RPC_USER}"), + &format!("--bitcoind.rpcpass={BTC_RPC_PASS}"), + &format!("--bitcoind.zmqpubrawblock=tcp://{bitcoind_name}:{ZMQ_RAWBLOCK_PORT}"), + &format!("--bitcoind.zmqpubrawtx=tcp://{bitcoind_name}:{ZMQ_RAWTX_PORT}"), + &format!("--rpclisten=0.0.0.0:{GRPC_PORT}"), + &format!("--listen=0.0.0.0:{P2P_PORT}"), + "--accept-keysend", + &format!("--alias={ALIAS}"), + // Zero out routing fees so multi-hop payments through this node never hit + // sender-side fee budgets. + "--bitcoin.basefee=0", + "--bitcoin.feerate=0", + // The TLS cert is generated on first boot; make it validate both for other + // containers (by name) and for the host dialing the mapped port. + &format!("--tlsextradomain={name}"), + "--tlsextraip=127.0.0.1", + ]) + .start() + .await + .context("starting lnd container")?; + + let init = async { + let cert_path = creds_dir.join("lnd/tls.cert"); + let macaroon_path = creds_dir.join("lnd/admin.macaroon"); + extract_file( + &container, + "/root/.lnd/tls.cert", + &cert_path, + Backoff::slow(), + ) + .await?; + extract_file( + &container, + "/root/.lnd/data/chain/bitcoin/regtest/admin.macaroon", + &macaroon_path, + Backoff::slow(), + ) + .await?; + + let grpc_host_port = container.get_host_port_ipv4(GRPC_PORT).await?; + let address = format!("https://127.0.0.1:{grpc_host_port}"); + + let client = with_backoff("lnd grpc connect", Backoff::slow(), || async { + tonic_lnd::connect(address.clone(), &cert_path, &macaroon_path).await + }) + .await?; + + let info = with_backoff("lnd synced to chain", Backoff::slow(), || { + let mut client = client.clone(); + async move { + let info = client + .lightning() + .get_info(lnrpc::GetInfoRequest {}) + .await + .map_err(|e| anyhow!(e.to_string()))? + .into_inner(); + if info.synced_to_chain { + Ok(info) + } else { + Err(anyhow!("not yet synced to chain")) + } + } + }) + .await?; + + let pubkey = info.identity_pubkey.parse().context("parsing lnd pubkey")?; + anyhow::Ok((client, pubkey, grpc_host_port, cert_path, macaroon_path)) + }; + + match init.await { + Ok((client, pubkey, grpc_host_port, cert_path, macaroon_path)) => Ok(LndHarness { + container, + name, + pubkey, + client, + grpc_host_port, + cert_path, + macaroon_path, + }), + Err(e) => { + dump_logs("lnd", &container).await; + Err(e.context("lnd startup")) + }, + } + } + + pub fn container(&self) -> &ContainerAsync { + &self.container + } + + pub fn p2p_address(&self) -> String { + format!("{}:{P2P_PORT}", self.name) + } + + pub fn node_handle(&self) -> NodeHandle { + NodeHandle { + pubkey: self.pubkey, + alias: ALIAS.to_string(), + implementation: NodeImpl::Lnd, + } + } + + /// The `nodes` entry for sim.json, connecting from the host through the mapped port. + pub fn sim_config(&self) -> Value { + json!({ + "id": self.pubkey.to_string(), + "address": format!("https://127.0.0.1:{}", self.grpc_host_port), + "macaroon": self.macaroon_path.to_string_lossy(), + "cert": self.cert_path.to_string_lossy(), + }) + } + + pub async fn new_address(&self) -> anyhow::Result { + let mut client = self.client.clone(); + let response = client + .lightning() + .new_address(lnrpc::NewAddressRequest { + r#type: lnrpc::AddressType::TaprootPubkey as i32, + ..Default::default() + }) + .await? + .into_inner(); + Ok(response.address) + } + + /// Connects to the peer and opens an announced channel, retrying until the wallet has seen + /// its funding confirm. + pub async fn open_channel(&self, peer: &PublicKey, peer_addr: &str) -> anyhow::Result<()> { + with_backoff("lnd open channel", Backoff::slow(), || async { + let mut client = self.client.clone(); + // Connecting to an already-connected peer errors; the open below is the real check. + let _ = client + .lightning() + .connect_peer(lnrpc::ConnectPeerRequest { + addr: Some(lnrpc::LightningAddress { + pubkey: peer.to_string(), + host: peer_addr.to_string(), + }), + ..Default::default() + }) + .await; + + client + .lightning() + .open_channel_sync(lnrpc::OpenChannelRequest { + node_pubkey: peer.serialize().to_vec(), + local_funding_amount: CHANNEL_SIZE_SAT as i64, + ..Default::default() + }) + .await + .map_err(|e| anyhow!(e.to_string())) + }) + .await?; + Ok(()) + } + + pub async fn active_channel_count(&self) -> anyhow::Result { + let mut client = self.client.clone(); + let response = client + .lightning() + .list_channels(lnrpc::ListChannelsRequest { + active_only: true, + ..Default::default() + }) + .await? + .into_inner(); + Ok(response.channels.len()) + } + + /// Checks that this node's graph view has at least the given node and channel counts. + pub async fn graph_synced(&self, nodes: usize, channels: usize) -> anyhow::Result<()> { + let mut client = self.client.clone(); + let graph = client + .lightning() + .describe_graph(lnrpc::ChannelGraphRequest { + include_unannounced: false, + }) + .await? + .into_inner(); + if graph.nodes.len() >= nodes && graph.edges.len() >= channels { + Ok(()) + } else { + Err(anyhow!( + "graph has {}/{nodes} nodes, {}/{channels} channels", + graph.nodes.len(), + graph.edges.len() + )) + } + } + + pub async fn settled_keysend_count(&self) -> anyhow::Result { + let mut client = self.client.clone(); + let response = client + .lightning() + .list_invoices(lnrpc::ListInvoiceRequest { + num_max_invoices: 10_000, + ..Default::default() + }) + .await? + .into_inner(); + Ok(response + .invoices + .iter() + .filter(|i| i.is_keysend && i.state == lnrpc::invoice::InvoiceState::Settled as i32) + .count() as u64) + } +} diff --git a/integration-tests/src/env/containers/mod.rs b/integration-tests/src/env/containers/mod.rs new file mode 100644 index 00000000..c943f194 --- /dev/null +++ b/integration-tests/src/env/containers/mod.rs @@ -0,0 +1,268 @@ +//! A heterogeneous real-node network run in docker containers: bitcoind plus one node each of +//! LND, CLN, Eclair and ldk-server, connected in a ring of announced channels. +//! +//! Startup is error resistant by construction: every step that depends on a node becoming ready +//! polls with backoff (see [`crate::retry`]) rather than assuming readiness, and each harness +//! dumps its container's logs when its own startup fails so errors point at the node that never +//! came up. + +pub mod bitcoind; +pub mod cln; +pub mod eclair; +pub mod ldk_server; +pub mod lnd; + +use std::path::Path; + +use anyhow::{anyhow, Context}; +use serde_json::Value; +use testcontainers::core::{CmdWaitFor, ExecCommand}; +use testcontainers::{ContainerAsync, GenericImage}; + +use super::{ConfigFragment, NodeHandle, TestNetwork}; +use crate::retry::{with_backoff, Backoff}; + +/// The p2p listening port used by every lightning node container. +pub const P2P_PORT: u16 = 9735; + +/// Size of every channel in the ring. +pub const CHANNEL_SIZE_SAT: u64 = 1_000_000; + +/// On-chain amount each channel-opening node is funded with. +const ONCHAIN_FUND_SAT: u64 = 10_000_000; + +/// Shared credentials for bitcoind RPC, used by every node's chain backend connection. +pub const BTC_RPC_USER: &str = "user"; +pub const BTC_RPC_PASS: &str = "pass"; + +/// Runs a command in a container and returns its raw stdout, failing on non-zero exit. +pub(crate) async fn exec_stdout( + container: &ContainerAsync, + cmd: &[&str], +) -> anyhow::Result> { + let mut result = container + .exec( + ExecCommand::new(cmd.iter().copied()) + .with_cmd_ready_condition(CmdWaitFor::exit_code(0)), + ) + .await + .with_context(|| format!("exec {cmd:?}"))?; + result + .stdout_to_vec() + .await + .with_context(|| format!("reading stdout of {cmd:?}")) +} + +/// Copies a file out of a container into `dest` on the host, polling until it exists and is +/// non-empty (files like macaroons and certs are created asynchronously on node startup). +pub(crate) async fn extract_file( + container: &ContainerAsync, + container_path: &str, + dest: &Path, + backoff: Backoff, +) -> anyhow::Result> { + let contents = with_backoff( + &format!("waiting for {container_path} in container"), + backoff, + || async { + match exec_stdout(container, &["cat", container_path]).await { + Ok(bytes) if !bytes.is_empty() => Ok(bytes), + Ok(_) => Err(anyhow!("{container_path} exists but is empty")), + Err(e) => Err(e), + } + }, + ) + .await?; + + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(dest, &contents) + .with_context(|| format!("writing {} to host", dest.display()))?; + Ok(contents) +} + +/// Prints the tail of a container's logs, used when a node fails to become ready or a scenario +/// fails, so CI output includes the node-side view of the failure. +pub(crate) async fn dump_logs(name: &str, container: &ContainerAsync) { + for (stream, bytes) in [ + ("stdout", container.stdout_to_vec().await), + ("stderr", container.stderr_to_vec().await), + ] { + let text = match bytes { + Ok(b) => String::from_utf8_lossy(&b).into_owned(), + Err(e) => format!(""), + }; + let tail: Vec<&str> = text.lines().rev().take(100).collect(); + eprintln!("===== {name} {stream} (last {} lines) =====", tail.len()); + for line in tail.iter().rev() { + eprintln!("{line}"); + } + } +} + +/// The full heterogeneous network. Nodes are indexed in ring order: 0 = LND, 1 = CLN, +/// 2 = Eclair, 3 = ldk-server, with channels 0->1->2->3->0. +pub struct RealNetwork { + handles: Vec, + node_configs: Vec, + pub bitcoind: bitcoind::Bitcoind, + pub lnd: lnd::LndHarness, + pub cln: cln::ClnHarness, + pub eclair: eclair::EclairHarness, + pub ldk: ldk_server::LdkHarness, +} + +impl RealNetwork { + /// Brings up the whole network: bitcoind, all four nodes in parallel, on-chain funding, the + /// ring of channels, and finally waits for every node to see the full graph. Extracted + /// credentials (certs, macaroons) are written under `creds_dir`, which must outlive the + /// simulations run against this network. + pub async fn start(creds_dir: &Path) -> anyhow::Result { + let docker_network = format!("simln-itest-{}", std::process::id()); + + let bitcoind = bitcoind::Bitcoind::start(&docker_network).await?; + + // The four nodes are independent of each other until channels open: start them in + // parallel since (particularly with cold image pulls) startup dominates runtime. + let (lnd, cln, eclair, ldk) = tokio::try_join!( + lnd::LndHarness::start(&docker_network, bitcoind.container_name(), creds_dir), + cln::ClnHarness::start(&docker_network, bitcoind.container_name(), creds_dir), + eclair::EclairHarness::start(&docker_network, bitcoind.container_name()), + ldk_server::LdkHarness::start(&docker_network, bitcoind.container_name(), creds_dir), + )?; + + // Fund every node that opens a channel from its own wallet. Eclair spends directly from + // bitcoind's wallet, which is already funded from mining. + let addresses = [ + lnd.new_address().await?, + cln.new_address().await?, + ldk.new_address().await?, + ]; + bitcoind.fund(&addresses, ONCHAIN_FUND_SAT)?; + + // Open the ring. Each open polls with backoff because it can only succeed once the + // opener's wallet has seen the funding confirmation. + lnd.open_channel(&cln.pubkey, &cln.p2p_address()).await?; + cln.open_channel(&eclair.pubkey, eclair.container_name(), P2P_PORT) + .await?; + eclair.open_channel(&ldk.pubkey, &ldk.p2p_address()).await?; + ldk.open_channel(&lnd.pubkey, &lnd.p2p_address()).await?; + + // Confirm the funding transactions deeply enough for the channels to be announced. Some + // implementations broadcast their funding transaction asynchronously after the open call + // returns (ldk-node batches broadcasts), so await_ready keeps mining while it waits + // rather than relying on this one round of confirmations. + bitcoind.mine(6)?; + + let result = Self::await_ready(&bitcoind, &lnd, &cln, &eclair, &ldk).await; + let network = RealNetwork { + handles: vec![ + lnd.node_handle(), + cln.node_handle(), + eclair.node_handle(), + ldk.node_handle(), + ], + node_configs: vec![ + lnd.sim_config(), + cln.sim_config(), + eclair.sim_config(), + ldk.sim_config(), + ], + bitcoind, + lnd, + cln, + eclair, + ldk, + }; + + if let Err(e) = result { + network.dump_all_logs().await; + return Err(e); + } + Ok(network) + } + + /// Waits until every node reports both of its ring channels active, and until every node's + /// own gossip view contains the full network (4 channels, 4 node announcements). Gossip + /// propagation is the slowest and least predictable step, so it gets the long schedule. + async fn await_ready( + bitcoind: &bitcoind::Bitcoind, + lnd: &lnd::LndHarness, + cln: &cln::ClnHarness, + eclair: &eclair::EclairHarness, + ldk: &ldk_server::LdkHarness, + ) -> anyhow::Result<()> { + with_backoff("ring channels active", Backoff::slow(), || async { + // Nudge any funding transaction that was broadcast after the initial confirmation + // round towards confirmation; extra regtest blocks are harmless. + bitcoind.mine(1)?; + + let (lnd_active, cln_active, eclair_active, ldk_active) = tokio::try_join!( + lnd.active_channel_count(), + cln.active_channel_count(), + eclair.active_channel_count(), + ldk.channel_count(), + )?; + if [lnd_active, cln_active, eclair_active, ldk_active] == [2, 2, 2, 2] { + Ok(()) + } else { + Err(anyhow!( + "channels active: lnd {lnd_active}/2, cln {cln_active}/2, \ + eclair {eclair_active}/2, ldk {ldk_active}/2" + )) + } + }) + .await?; + + with_backoff("lnd graph sync", Backoff::slow(), || async { + lnd.graph_synced(4, 4).await + }) + .await?; + with_backoff("cln graph sync", Backoff::slow(), || async { + cln.graph_synced(4, 4).await + }) + .await?; + with_backoff("eclair graph sync", Backoff::slow(), || async { + eclair.graph_synced(4, 4).await + }) + .await?; + with_backoff("ldk-server graph sync", Backoff::slow(), || async { + ldk.graph_synced(4, 4).await + }) + .await?; + + Ok(()) + } + + /// The number of settled inbound keysend payments the node at ring index `idx` reports, from + /// the node's own books — used to verify receipt independently of sim-ln's records. + pub async fn settled_keysend_count(&self, idx: usize) -> anyhow::Result { + match idx { + 0 => self.lnd.settled_keysend_count().await, + 1 => self.cln.settled_keysend_count().await, + 2 => self.eclair.settled_keysend_count().await, + 3 => self.ldk.settled_keysend_count().await, + n => Err(anyhow!("no node at ring index {n}")), + } + } + + /// Dumps the tail of every container's logs to test output. + pub async fn dump_all_logs(&self) { + dump_logs("bitcoind", self.bitcoind.container()).await; + dump_logs("lnd", self.lnd.container()).await; + dump_logs("cln", self.cln.container()).await; + dump_logs("eclair", self.eclair.container()).await; + dump_logs("ldk-server", self.ldk.container()).await; + } +} + +impl TestNetwork for RealNetwork { + fn nodes(&self) -> &[NodeHandle] { + &self.handles + } + + fn config_fragment(&self) -> ConfigFragment { + ConfigFragment::RealNodes(self.node_configs.clone()) + } +} diff --git a/integration-tests/src/env/mod.rs b/integration-tests/src/env/mod.rs new file mode 100644 index 00000000..34103722 --- /dev/null +++ b/integration-tests/src/env/mod.rs @@ -0,0 +1,45 @@ +//! The environment layer: provisions a lightning network and describes it as a partial simulation +//! config. Implementations know nothing about the payment activity that will run on the network. + +pub mod containers; +pub mod simulated; + +use bitcoin::secp256k1::PublicKey; + +/// The node implementation backing a [`NodeHandle`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NodeImpl { + Simulated, + Lnd, + Cln, + Eclair, + LdkServer, +} + +/// A node in a test network, exposing just enough for scenarios to reference it in config and for +/// assertions to identify it in results. +#[derive(Debug, Clone)] +pub struct NodeHandle { + pub pubkey: PublicKey, + pub alias: String, + pub implementation: NodeImpl, +} + +/// The environment's contribution to a sim.json file. Built as raw JSON rather than the parsing +/// crate's own types so that tests exercise real deserialization of the file format. +#[derive(Debug, Clone)] +pub enum ConfigFragment { + /// Entries for the `nodes` key: connection details for real nodes. + RealNodes(Vec), + /// Entries for the `sim_network` key: channels of a simulated network. + SimGraph(Vec), +} + +/// A provisioned network that simulations can run against. +pub trait TestNetwork { + /// The nodes that the simulation will control. + fn nodes(&self) -> &[NodeHandle]; + + /// The network's contribution to the simulation config. + fn config_fragment(&self) -> ConfigFragment; +} diff --git a/integration-tests/src/env/simulated.rs b/integration-tests/src/env/simulated.rs new file mode 100644 index 00000000..b65b853c --- /dev/null +++ b/integration-tests/src/env/simulated.rs @@ -0,0 +1,101 @@ +//! An in-process simulated network: a deterministic graph emitted as a `sim_network` config +//! section, requiring no external processes. + +use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey}; +use serde_json::{json, Value}; + +use super::{ConfigFragment, NodeHandle, NodeImpl, TestNetwork}; + +/// Capacity of every channel in the simulated network. Large relative to test payment amounts so +/// that liquidity never limits a test scenario. +pub const CHANNEL_CAPACITY_MSAT: u64 = 100_000_000; + +/// A deterministic five-node simulated network: a ring of nodes 0-1-2-3 plus a spur node 4 +/// attached to node 0, so every pair of nodes is connected and some pairs (e.g. 4 -> 2) can only +/// be reached over multiple hops. +/// +/// Fees are set to zero on all channels so that payment amounts observed in results exactly match +/// the amounts scenarios dispatch, and routing fee budgets can never interfere with tests that are +/// not about fees. +pub struct SimulatedNetwork { + handles: Vec, + channels: Vec, +} + +impl SimulatedNetwork { + /// The channel endpoints of the network, as indices into the node list. + const EDGES: [(usize, usize); 5] = [(0, 1), (1, 2), (2, 3), (3, 0), (0, 4)]; + + pub fn new() -> Self { + Self::with_aliases(&["node_0", "node_1", "node_2", "node_3", "node_4"]) + } + + /// A network where two nodes share an alias, for tests asserting that duplicate aliases are + /// rejected. + pub fn with_duplicate_alias() -> Self { + Self::with_aliases(&["node_0", "node_1", "node_2", "node_3", "node_3"]) + } + + fn with_aliases(aliases: &[&str]) -> Self { + let secp = Secp256k1::new(); + let handles: Vec = aliases + .iter() + .enumerate() + .map(|(i, alias)| { + // Deterministic keys so that runs are reproducible and pubkey-referenced configs + // can be asserted against. + let secret = SecretKey::from_slice(&[i as u8 + 1; 32]).expect("static key valid"); + NodeHandle { + pubkey: PublicKey::from_secret_key(&secp, &secret), + alias: alias.to_string(), + implementation: NodeImpl::Simulated, + } + }) + .collect(); + + let channels = Self::EDGES + .iter() + .enumerate() + .map(|(i, (a, b))| { + json!({ + "scid": i as u64 + 1, + "capacity_msat": CHANNEL_CAPACITY_MSAT, + "node_1": Self::policy(&handles[*a]), + "node_2": Self::policy(&handles[*b]), + }) + }) + .collect(); + + SimulatedNetwork { handles, channels } + } + + fn policy(node: &NodeHandle) -> Value { + json!({ + "pubkey": node.pubkey.to_string(), + "alias": node.alias, + "max_htlc_count": 483, + "max_in_flight_msat": CHANNEL_CAPACITY_MSAT, + "min_htlc_size_msat": 1, + "max_htlc_size_msat": CHANNEL_CAPACITY_MSAT, + "cltv_expiry_delta": 40, + "base_fee": 0, + "fee_rate_prop": 0, + }) + } +} + +impl Default for SimulatedNetwork { + fn default() -> Self { + Self::new() + } +} + +impl TestNetwork for SimulatedNetwork { + fn nodes(&self) -> &[NodeHandle] { + &self.handles + } + + fn config_fragment(&self) -> ConfigFragment { + ConfigFragment::SimGraph(self.channels.clone()) + } +} diff --git a/integration-tests/src/lib.rs b/integration-tests/src/lib.rs new file mode 100644 index 00000000..e9cceed3 --- /dev/null +++ b/integration-tests/src/lib.rs @@ -0,0 +1,17 @@ +//! Integration test framework for sim-ln. +//! +//! The framework is split into independent layers: +//! - [`env`]: provisions a lightning network (simulated or real) and describes it as a partial +//! simulation config, without any knowledge of the payments that will run on it. +//! - [`scenario`]: describes payment activity (defined or random) and the style in which it is +//! written to config (aliases vs pubkeys, scalar vs range values), without any knowledge of how +//! the underlying network is provisioned. +//! - [`runner`]: assembles a sim.json file from the two layers, runs it through the same public +//! entry points the sim-cli binary uses, and collects observable output. +//! - [`asserts`]: shared assertions over that output. + +pub mod asserts; +pub mod env; +pub mod retry; +pub mod runner; +pub mod scenario; diff --git a/integration-tests/src/retry.rs b/integration-tests/src/retry.rs new file mode 100644 index 00000000..7d2b90ac --- /dev/null +++ b/integration-tests/src/retry.rs @@ -0,0 +1,117 @@ +//! Retry with capped exponential backoff, used to make real-node startup error resistant: nodes +//! and their RPCs come up at unpredictable times, so every setup step polls rather than assuming +//! readiness. + +use std::fmt::Display; +use std::future::Future; +use std::time::Duration; + +use anyhow::anyhow; +use tokio::time::Instant; + +/// Backoff schedule for [`with_backoff`]. Delays double from `initial` up to `max`, and the +/// operation as a whole fails once `timeout` has elapsed. +#[derive(Debug, Clone, Copy)] +pub struct Backoff { + pub initial: Duration, + pub max: Duration, + pub timeout: Duration, +} + +impl Default for Backoff { + fn default() -> Self { + Backoff { + initial: Duration::from_millis(250), + max: Duration::from_secs(5), + timeout: Duration::from_secs(60), + } + } +} + +impl Backoff { + /// A schedule for operations that are expected to take a while to converge, such as waiting + /// for gossip to propagate through a network. + pub fn slow() -> Self { + Backoff { + initial: Duration::from_secs(1), + max: Duration::from_secs(10), + timeout: Duration::from_secs(300), + } + } +} + +/// Runs `op` until it succeeds or `backoff.timeout` elapses, sleeping between attempts. The +/// returned error names the operation and includes the last underlying error, so failures point +/// at the step (and node) that never became ready. +pub async fn with_backoff( + description: &str, + backoff: Backoff, + mut op: F, +) -> Result +where + E: Display, + F: FnMut() -> Fut, + Fut: Future>, +{ + let start = Instant::now(); + let mut delay = backoff.initial; + + loop { + match op().await { + Ok(t) => return Ok(t), + Err(e) => { + if start.elapsed() + delay > backoff.timeout { + return Err(anyhow!( + "{description}: not ready after {:?}, last error: {e}", + start.elapsed() + )); + } + + log::debug!("{description}: retrying in {delay:?} after error: {e}"); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(backoff.max); + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU32, Ordering}; + + #[tokio::test] + async fn test_succeeds_after_failures() { + let attempts = AtomicU32::new(0); + let result = with_backoff("test op", Backoff::default(), || async { + if attempts.fetch_add(1, Ordering::SeqCst) < 2 { + Err("not yet") + } else { + Ok(42) + } + }) + .await + .unwrap(); + + assert_eq!(result, 42); + assert_eq!(attempts.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn test_times_out_with_context() { + let backoff = Backoff { + initial: Duration::from_millis(1), + max: Duration::from_millis(2), + timeout: Duration::from_millis(20), + }; + let err = with_backoff("flaky node startup", backoff, || async { + Err::<(), _>("connection refused") + }) + .await + .unwrap_err(); + + let msg = err.to_string(); + assert!(msg.contains("flaky node startup")); + assert!(msg.contains("connection refused")); + } +} diff --git a/integration-tests/src/runner.rs b/integration-tests/src/runner.rs new file mode 100644 index 00000000..7d30c742 --- /dev/null +++ b/integration-tests/src/runner.rs @@ -0,0 +1,269 @@ +//! The execution layer: assembles sim.json files from the environment and scenario layers, runs +//! them through the same public entry points the sim-cli binary uses, and collects the observable +//! output for assertions. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::sync::Arc; +use std::time::SystemTime; + +use anyhow::{anyhow, Context}; +use bitcoin::secp256k1::PublicKey; +use serde_json::{json, Value}; +use sim_cli::parsing::{create_simulation, create_simulation_with_network, SimParams}; +use simln_lib::clock::SimulationClock; +use simln_lib::runtime::block_on_virtual_time; +use simln_lib::sim_node::CustomRecords; +use simln_lib::{ActivityDefinition, Simulation, SimulationCfg, WriteResults}; +use tokio_util::task::TaskTracker; + +use crate::env::{ConfigFragment, NodeHandle}; +use crate::scenario::Scenario; + +/// Assembles a sim.json file from the environment and scenario layers. +pub struct SimFile { + fragment: ConfigFragment, + activity: Vec, + exclude: Vec, +} + +impl SimFile { + pub fn new(fragment: ConfigFragment) -> Self { + SimFile { + fragment, + activity: vec![], + exclude: vec![], + } + } + + /// Populates activity and exclusions from a scenario. + pub fn scenario(mut self, scenario: &Scenario, nodes: &[NodeHandle]) -> Self { + self.activity = scenario.activity_json(nodes); + self.exclude = scenario.exclude_json(nodes); + self + } + + /// Sets a raw activity section, for tests that intentionally write invalid references. + pub fn activity_raw(mut self, activity: Vec) -> Self { + self.activity = activity; + self + } + + /// Writes the assembled sim.json into `dir` and returns its path. + pub fn write(self, dir: &Path) -> anyhow::Result { + let mut config = json!({}); + match self.fragment { + ConfigFragment::RealNodes(nodes) => config["nodes"] = json!(nodes), + ConfigFragment::SimGraph(channels) => config["sim_network"] = json!(channels), + } + if !self.activity.is_empty() { + config["activity"] = json!(self.activity); + } + if !self.exclude.is_empty() { + config["exclude"] = json!(self.exclude); + } + + let path = dir.join("sim.json"); + std::fs::write(&path, serde_json::to_string_pretty(&config)?) + .with_context(|| format!("writing sim file to {}", path.display()))?; + Ok(path) + } +} + +/// Options controlling a simulation run, mirroring the sim-cli flags relevant to tests. +#[derive(Debug, Clone, Copy)] +pub struct RunOptions { + pub total_time: Option, + pub expected_pmt_amt: u64, + pub capacity_multiplier: f64, + pub fix_seed: Option, +} + +impl Default for RunOptions { + fn default() -> Self { + RunOptions { + total_time: Some(3600), + expected_pmt_amt: 3_800_000, + capacity_multiplier: 2.0, + fix_seed: Some(42), + } + } +} + +/// A payment recorded in the simulation's results CSV. +#[derive(Debug, Clone)] +pub struct PaymentRecord { + pub source: PublicKey, + pub destination: PublicKey, + pub amount_msat: u64, + pub outcome: String, +} + +impl PaymentRecord { + pub fn is_success(&self) -> bool { + self.outcome == "Success" + } +} + +/// The observable output of a simulation run. +#[derive(Debug)] +pub struct SimOutput { + /// The parameters as deserialized from the sim file, for assertions on parsing itself. + pub params: SimParams, + /// The validated activities the simulation ran with, with node references fully resolved. + pub activities: Vec, + /// Total payments dispatched, as reported by the simulation. + pub total_payments: u64, + /// Success rate percentage, as reported by the simulation. + pub success_rate: f64, + /// Per-payment records read back from the results CSV. + pub records: Vec, +} + +/// Deserializes a sim file exactly as the sim-cli binary would. +pub fn parse_params(sim_file: &Path) -> anyhow::Result { + let contents = std::fs::read_to_string(sim_file) + .with_context(|| format!("reading sim file {}", sim_file.display()))?; + serde_json::from_str(&contents).context("deserializing sim file") +} + +/// Runs a sim file describing a simulated network to completion on virtual time, so runs bounded +/// by `total_time` finish as fast as the CPU allows. Must be called from a synchronous context +/// (not inside a tokio runtime). +pub fn run_simulated(sim_file: &Path, opts: RunOptions) -> anyhow::Result { + let params = parse_params(sim_file)?; + let (cfg, results_dir) = simulation_cfg(sim_file, opts)?; + + let run_params = params.clone(); + // Anchor virtual time at the wall clock: the simulated graph's channel updates are stamped + // with clock time, and pathfinding rejects gossip older than two weeks. Reproducibility of + // payment sequences comes from the seeded RNG, not the clock anchor. + let (activities, total_payments, success_rate) = + block_on_virtual_time(SystemTime::now(), |clock| async move { + let (sim, activities, _nodes) = create_simulation_with_network( + cfg, + &run_params, + clock, + TaskTracker::new(), + vec![], + CustomRecords::default(), + ) + .await?; + + finish(sim, &activities) + .await + .map(|(total, rate)| (activities, total, rate)) + })??; + + Ok(SimOutput { + params, + activities, + total_payments, + success_rate, + records: read_records(&results_dir)?, + }) +} + +/// Runs a sim file describing real nodes to completion on wall-clock time, from within a tokio +/// runtime. +pub async fn run_real(sim_file: &Path, opts: RunOptions) -> anyhow::Result { + let params = parse_params(sim_file)?; + let (cfg, results_dir) = simulation_cfg(sim_file, opts)?; + + let clock = Arc::new(SimulationClock::new(SystemTime::now())); + let (sim, activities) = create_simulation(cfg, ¶ms, clock, TaskTracker::new()).await?; + let (total_payments, success_rate) = finish(sim, &activities).await?; + + Ok(SimOutput { + params, + activities, + total_payments, + success_rate, + records: read_records(&results_dir)?, + }) +} + +/// Builds the simulation config, creating a results directory next to the sim file so each test +/// run's CSV output is isolated with the rest of its files. +fn simulation_cfg(sim_file: &Path, opts: RunOptions) -> anyhow::Result<(SimulationCfg, PathBuf)> { + let results_dir = sim_file + .parent() + .ok_or_else(|| anyhow!("sim file {} has no parent directory", sim_file.display()))? + .join("results"); + std::fs::create_dir_all(&results_dir)?; + + let cfg = SimulationCfg::new( + opts.total_time, + opts.expected_pmt_amt, + opts.capacity_multiplier, + Some(WriteResults { + results_dir: results_dir.clone(), + // Flush every record so results are complete even if shutdown races the writer. + batch_size: 1, + }), + opts.fix_seed, + ); + + Ok((cfg, results_dir)) +} + +/// Drives a configured simulation to completion and reports its aggregates. +async fn finish( + sim: Simulation, + activities: &[ActivityDefinition], +) -> anyhow::Result<(u64, f64)> { + sim.run(activities).await?; + Ok((sim.get_total_payments().await, sim.get_success_rate().await)) +} + +/// Reads back the payment records written to the results directory. The CSV rows are flattened +/// `(Payment, PaymentResult)` tuples with a header row: source, destination, amount_msat, hash, +/// dispatch_time, htlc_count, payment_outcome. +fn read_records(results_dir: &Path) -> anyhow::Result> { + let mut csv_files: Vec = std::fs::read_dir(results_dir)? + .filter_map(|entry| { + let path = entry.ok()?.path(); + (path.extension()? == "csv").then_some(path) + }) + .collect(); + + let path = match csv_files.len() { + 0 => return Err(anyhow!("no results CSV found in {}", results_dir.display())), + 1 => csv_files.remove(0), + n => { + return Err(anyhow!( + "expected a single results CSV in {}, found {n}", + results_dir.display() + )) + }, + }; + + let mut reader = csv::ReaderBuilder::new() + .has_headers(true) + .from_path(&path)?; + + let mut records = vec![]; + for row in reader.records() { + let row = row?; + let field = |i: usize| -> anyhow::Result<&str> { + row.get(i) + .ok_or_else(|| anyhow!("results row missing field {i}: {row:?}")) + }; + + records.push(PaymentRecord { + source: PublicKey::from_str(field(0)?).context("results row source pubkey")?, + destination: PublicKey::from_str(field(1)?) + .context("results row destination pubkey")?, + amount_msat: field(2)?.parse().context("results row amount")?, + outcome: field(6)?.to_string(), + }); + } + + Ok(records) +} + +/// Convenience lookup from pubkey to handle for assertion messages. +pub fn handles_by_pubkey(nodes: &[NodeHandle]) -> HashMap { + nodes.iter().map(|n| (n.pubkey, n)).collect() +} diff --git a/integration-tests/src/scenario.rs b/integration-tests/src/scenario.rs new file mode 100644 index 00000000..8df63a44 --- /dev/null +++ b/integration-tests/src/scenario.rs @@ -0,0 +1,122 @@ +//! The scenario layer: describes payment activity and the style in which it is written to config. +//! Scenarios reference nodes by index into a [`NodeHandle`] slice, so they are independent of how +//! the underlying network is provisioned. + +use serde_json::{json, Value}; + +use crate::env::NodeHandle; + +/// How a node is referenced in the config file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NodeRef { + Alias, + Pubkey, +} + +impl NodeRef { + fn to_json(self, node: &NodeHandle) -> Value { + match self { + NodeRef::Alias => json!(node.alias), + NodeRef::Pubkey => json!(node.pubkey.to_string()), + } + } +} + +/// A config value that is either a scalar or a `[min, max]` range. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ValueSpec { + Scalar(u64), + Range(u64, u64), +} + +impl ValueSpec { + pub fn to_json(self) -> Value { + match self { + ValueSpec::Scalar(v) => json!(v), + ValueSpec::Range(min, max) => json!([min, max]), + } + } + + /// Whether a concrete value could have been produced from this spec. + pub fn contains(self, value: u64) -> bool { + match self { + ValueSpec::Scalar(v) => value == v, + ValueSpec::Range(min, max) => (min..max).contains(&value), + } + } +} + +/// The style in which defined activity is written to config: which identifier type references +/// nodes, and whether amounts/intervals are scalars or ranges. +#[derive(Debug, Clone, Copy)] +pub struct ConfigStyle { + pub source_ref: NodeRef, + pub dest_ref: NodeRef, + pub amount_msat: ValueSpec, + pub interval_secs: ValueSpec, +} + +impl Default for ConfigStyle { + fn default() -> Self { + ConfigStyle { + source_ref: NodeRef::Pubkey, + dest_ref: NodeRef::Pubkey, + amount_msat: ValueSpec::Scalar(1000), + interval_secs: ValueSpec::Scalar(2), + } + } +} + +/// Payment activity to run on a network. +#[derive(Debug, Clone)] +pub enum Scenario { + /// Defined activity between pairs of nodes (indices into the handle slice), each dispatching + /// `count` payments (or running until the simulation's total time when `None`). + Defined { + pairs: Vec<(usize, usize)>, + count: Option, + style: ConfigStyle, + }, + /// Random activity across the network, excluding the given nodes (indices into the handle + /// slice) from sending and receiving. + Random { excludes: Vec }, +} + +impl Scenario { + /// The `activity` section of the config; empty for random activity. + pub fn activity_json(&self, nodes: &[NodeHandle]) -> Vec { + match self { + Scenario::Defined { + pairs, + count, + style, + } => pairs + .iter() + .map(|(source, dest)| { + let mut activity = json!({ + "source": style.source_ref.to_json(&nodes[*source]), + "destination": style.dest_ref.to_json(&nodes[*dest]), + "interval_secs": style.interval_secs.to_json(), + "amount_msat": style.amount_msat.to_json(), + }); + if let Some(count) = count { + activity["count"] = json!(count); + } + activity + }) + .collect(), + Scenario::Random { .. } => vec![], + } + } + + /// The `exclude` section of the config; empty for defined activity. + pub fn exclude_json(&self, nodes: &[NodeHandle]) -> Vec { + match self { + Scenario::Defined { .. } => vec![], + Scenario::Random { excludes } => excludes + .iter() + .map(|i| json!(nodes[*i].pubkey.to_string())) + .collect(), + } + } +} diff --git a/integration-tests/tests/real_nodes.rs b/integration-tests/tests/real_nodes.rs new file mode 100644 index 00000000..29065650 --- /dev/null +++ b/integration-tests/tests/real_nodes.rs @@ -0,0 +1,236 @@ +//! The real-node tier: a heterogeneous regtest network (LND, CLN, Eclair, ldk-server in a ring +//! of channels over one bitcoind) run in docker, with simulations driven against it through every +//! connector. +//! +//! Everything runs inside one test: container startup dominates runtime, so the network is built +//! once and the scenarios run sequentially against it. Containers are cleaned up when the test +//! ends because the network is dropped here — a shared static would leak them. + +use std::time::Duration; + +use anyhow::ensure; +use integration_tests::asserts::{ + assert_activities_resolved, assert_defined_payments, assert_payments_dispatched, + assert_sources_controlled, +}; +use integration_tests::env::containers::RealNetwork; +use integration_tests::env::TestNetwork; +use integration_tests::runner::{run_real, RunOptions, SimFile, SimOutput}; +use integration_tests::scenario::{ConfigStyle, NodeRef, Scenario, ValueSpec}; +use sim_cli::parsing::NodeConnection; + +/// Ring order: 0 = LND, 1 = CLN, 2 = Eclair, 3 = ldk-server; channels 0->1->2->3->0. +const RING: [(usize, usize); 4] = [(0, 1), (1, 2), (2, 3), (3, 0)]; + +#[tokio::test(flavor = "multi_thread")] +#[ignore = "requires docker; run with make integration-real"] +async fn real_nodes() { + let creds_dir = tempfile::tempdir().expect("create credentials dir"); + + eprintln!("=== starting heterogeneous regtest network (bitcoind + lnd/cln/eclair/ldk) ==="); + let network = RealNetwork::start(creds_dir.path()) + .await + .expect("real node network should start"); + eprintln!("=== network ready ==="); + + run_scenario( + "defined keysends around the ring (every connector sends and receives)", + &network, + scenario_ring_defined(&network), + ) + .await; + run_scenario( + "multi-hop defined payments (lnd -> cln -> eclair)", + &network, + scenario_multi_hop(&network), + ) + .await; + run_scenario( + "alias-referenced defined payments (cln -> ldk)", + &network, + scenario_alias_refs(&network), + ) + .await; + run_scenario( + "random activity across all four implementations", + &network, + scenario_random(&network), + ) + .await; +} + +/// Runs one scenario with a hard timeout, dumping every container's logs when it errors so CI +/// output includes the node-side view. (Assertion panics inside scenarios point at sim-side +/// state, which the panic message already carries.) +async fn run_scenario( + name: &str, + network: &RealNetwork, + scenario: impl std::future::Future>, +) { + eprintln!("=== scenario: {name} ==="); + match tokio::time::timeout(Duration::from_secs(600), scenario).await { + Ok(Ok(())) => eprintln!("=== scenario passed: {name} ==="), + Ok(Err(e)) => { + network.dump_all_logs().await; + panic!("scenario failed: {name}: {e:#}"); + }, + Err(_) => { + network.dump_all_logs().await; + panic!("scenario timed out: {name}"); + }, + } +} + +/// Writes the scenario to a sim file and runs it against the network. +async fn run( + network: &RealNetwork, + scenario: &Scenario, + opts: RunOptions, +) -> anyhow::Result { + let dir = tempfile::tempdir()?; + let sim_file = SimFile::new(network.config_fragment()) + .scenario(scenario, network.nodes()) + .write(dir.path())?; + run_real(&sim_file, opts).await +} + +/// One defined activity per directed ring edge: proves every connector both dispatches and +/// receives keysend cross-implementation, with receipt confirmed against each destination node's +/// own books rather than sim-ln's records alone. +async fn scenario_ring_defined(network: &RealNetwork) -> anyhow::Result<()> { + let count = 3u64; + + let mut receipts_before = [0u64; 4]; + for (i, receipts) in receipts_before.iter_mut().enumerate() { + *receipts = network.settled_keysend_count(i).await?; + } + + let scenario = Scenario::Defined { + pairs: RING.to_vec(), + count: Some(count), + style: ConfigStyle { + amount_msat: ValueSpec::Scalar(10_000), + interval_secs: ValueSpec::Scalar(2), + ..Default::default() + }, + }; + let out = run( + network, + &scenario, + RunOptions { + total_time: Some(180), + ..Default::default() + }, + ) + .await?; + + // The untagged `nodes` config section must infer each connector implementation from its + // fields alone. + ensure!( + matches!(out.params.nodes[0], NodeConnection::Lnd(_)) + && matches!(out.params.nodes[1], NodeConnection::Cln(_)) + && matches!(out.params.nodes[2], NodeConnection::Eclair(_)) + && matches!(out.params.nodes[3], NodeConnection::LdkServer(_)), + "node connection implementations were not inferred correctly from config" + ); + + let nodes = network.nodes(); + let pairs: Vec<_> = RING.iter().map(|(s, d)| (&nodes[*s], &nodes[*d])).collect(); + assert_activities_resolved(&out, &pairs); + assert_defined_payments(&out, &pairs, count); + + // Each ring destination's own node reports the keysends it received. The final payment of + // the run may still be settling (or its record lost to shutdown), hence the range. + for (_, dest) in RING { + let received = network.settled_keysend_count(dest).await? - receipts_before[dest]; + ensure!( + (count - 1..=count).contains(&received), + "node {} ({}) reports {received} received keysends, expected {} or {count}", + dest, + nodes[dest].alias, + count - 1, + ); + } + Ok(()) +} + +/// A pair with no direct channel: the payment must route across the ring, proving route +/// construction from real graph data. +async fn scenario_multi_hop(network: &RealNetwork) -> anyhow::Result<()> { + let count = 2u64; + let scenario = Scenario::Defined { + pairs: vec![(0, 2)], + count: Some(count), + style: ConfigStyle { + amount_msat: ValueSpec::Scalar(5000), + interval_secs: ValueSpec::Scalar(2), + ..Default::default() + }, + }; + let out = run( + network, + &scenario, + RunOptions { + total_time: Some(120), + ..Default::default() + }, + ) + .await?; + + let nodes = network.nodes(); + let pairs = [(&nodes[0], &nodes[2])]; + assert_activities_resolved(&out, &pairs); + assert_defined_payments(&out, &pairs, count); + Ok(()) +} + +/// Activities referencing nodes purely by alias, where the aliases come from the real nodes' +/// own configuration rather than a simulated graph description. +async fn scenario_alias_refs(network: &RealNetwork) -> anyhow::Result<()> { + let count = 2u64; + let scenario = Scenario::Defined { + pairs: vec![(1, 3)], + count: Some(count), + style: ConfigStyle { + source_ref: NodeRef::Alias, + dest_ref: NodeRef::Alias, + amount_msat: ValueSpec::Scalar(5000), + interval_secs: ValueSpec::Scalar(2), + }, + }; + let out = run( + network, + &scenario, + RunOptions { + total_time: Some(120), + ..Default::default() + }, + ) + .await?; + + let nodes = network.nodes(); + let pairs = [(&nodes[1], &nodes[3])]; + assert_activities_resolved(&out, &pairs); + assert_defined_payments(&out, &pairs, count); + Ok(()) +} + +/// Random activity across all four implementations, bounded by total time. +async fn scenario_random(network: &RealNetwork) -> anyhow::Result<()> { + let out = run( + network, + &Scenario::Random { excludes: vec![] }, + RunOptions { + total_time: Some(30), + // Small expected amounts and a raised multiplier so a 30s window sees payments. + expected_pmt_amt: 10_000, + capacity_multiplier: 5.0, + fix_seed: Some(42), + }, + ) + .await?; + + assert_payments_dispatched(&out); + assert_sources_controlled(&out, network.nodes()); + Ok(()) +} diff --git a/integration-tests/tests/sim_matrix.rs b/integration-tests/tests/sim_matrix.rs new file mode 100644 index 00000000..b27d6406 --- /dev/null +++ b/integration-tests/tests/sim_matrix.rs @@ -0,0 +1,276 @@ +//! The simulated-network test matrix: config-style and payment-modality coverage that is +//! backend-independent, run on virtual time so the whole matrix completes in seconds without +//! docker or external processes. + +use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey}; +use rstest::rstest; +use serde_json::json; +use sim_cli::parsing::NodeConnection; + +use integration_tests::asserts::{ + assert_activities_resolved, assert_all_success, assert_amounts_within, assert_defined_payments, + assert_not_involved, assert_payments_dispatched, assert_sources_controlled, + assert_total_payments, +}; +use integration_tests::env::simulated::SimulatedNetwork; +use integration_tests::env::TestNetwork; +use integration_tests::runner::{parse_params, run_simulated, RunOptions, SimFile, SimOutput}; +use integration_tests::scenario::{ConfigStyle, NodeRef, Scenario, ValueSpec}; + +/// Runs a scenario on a simulated network and returns its output, failing the test on any setup +/// or simulation error. +fn run(network: &SimulatedNetwork, scenario: &Scenario, opts: RunOptions) -> SimOutput { + let dir = tempfile::tempdir().expect("create temp dir"); + let sim_file = SimFile::new(network.config_fragment()) + .scenario(scenario, network.nodes()) + .write(dir.path()) + .expect("write sim file"); + run_simulated(&sim_file, opts).expect("simulation should run to completion") +} + +/// Runs a scenario expected to fail validation and returns the full error text. +fn run_expecting_error(network: &SimulatedNetwork, activity: Vec) -> String { + let dir = tempfile::tempdir().expect("create temp dir"); + let sim_file = SimFile::new(network.config_fragment()) + .activity_raw(activity) + .write(dir.path()) + .expect("write sim file"); + let err = run_simulated(&sim_file, RunOptions::default()) + .expect_err("simulation setup should fail validation"); + format!("{err:#}") +} + +/// Defined activity across every combination of node reference style and value shape: however the +/// config spells out nodes and values, the same payments must flow. Includes a single-hop pair +/// (node 0 -> 1) and a multi-hop pair (spur node 4 -> 2, which must route through the ring). +#[rstest] +#[ntest::timeout(120_000)] +fn defined_activity_config_styles( + #[values(NodeRef::Alias, NodeRef::Pubkey)] source_ref: NodeRef, + #[values(NodeRef::Alias, NodeRef::Pubkey)] dest_ref: NodeRef, + #[values(ValueSpec::Scalar(1000), ValueSpec::Range(1000, 10_000))] amount_msat: ValueSpec, + #[values(ValueSpec::Scalar(2), ValueSpec::Range(1, 5))] interval_secs: ValueSpec, +) { + let network = SimulatedNetwork::new(); + let count = 5; + let scenario = Scenario::Defined { + pairs: vec![(0, 1), (4, 2)], + count: Some(count), + style: ConfigStyle { + source_ref, + dest_ref, + amount_msat, + interval_secs, + }, + }; + + let out = run(&network, &scenario, RunOptions::default()); + + let nodes = network.nodes(); + let pairs = [(&nodes[0], &nodes[1]), (&nodes[4], &nodes[2])]; + assert_activities_resolved(&out, &pairs); + assert_amounts_within(&out, amount_msat); + assert_defined_payments(&out, &pairs, count); +} + +/// Defined activity without a count runs until the simulation's total time; on virtual time the +/// payment schedule is deterministic, so the exact number of dispatched payments is known. +#[rstest] +#[ntest::timeout(120_000)] +fn defined_activity_bounded_by_total_time() { + let network = SimulatedNetwork::new(); + let scenario = Scenario::Defined { + pairs: vec![(0, 1)], + count: None, + style: ConfigStyle { + interval_secs: ValueSpec::Scalar(10), + ..Default::default() + }, + }; + + let out = run( + &network, + &scenario, + RunOptions { + total_time: Some(95), + ..Default::default() + }, + ); + + // Payments dispatch every 10 virtual seconds until shutdown at t=95: t=10..=90. + assert_total_payments(&out, 9); + assert_all_success(&out); +} + +/// Random activity, with and without exclusions: payments flow between controlled nodes only, +/// and excluded nodes are never involved. +#[rstest] +#[case::no_exclusions(vec![])] +#[case::spur_node_excluded(vec![4])] +#[ntest::timeout(120_000)] +fn random_activity(#[case] excludes: Vec) { + let network = SimulatedNetwork::new(); + let scenario = Scenario::Random { + excludes: excludes.clone(), + }; + + let out = run( + &network, + &scenario, + RunOptions { + // A virtual day of activity with payments small relative to channel capacity. + total_time: Some(86_400), + expected_pmt_amt: 1_000_000, + ..Default::default() + }, + ); + + assert_payments_dispatched(&out); + assert_sources_controlled(&out, network.nodes()); + + let excluded: Vec<_> = excludes.iter().map(|i| &network.nodes()[*i]).collect(); + assert_not_involved(&out, &excluded); +} + +/// A seeded random run is reproducible: the same seed dispatches the identical payment sequence. +#[rstest] +#[ntest::timeout(120_000)] +fn random_activity_deterministic_with_seed() { + let opts = RunOptions { + total_time: Some(86_400), + expected_pmt_amt: 1_000_000, + fix_seed: Some(7), + ..Default::default() + }; + + let run_once = || { + let network = SimulatedNetwork::new(); + let out = run(&network, &Scenario::Random { excludes: vec![] }, opts); + out.records + .iter() + .map(|r| (r.source, r.destination, r.amount_msat)) + .collect::>() + }; + + let first = run_once(); + assert!(!first.is_empty()); + assert_eq!( + first, + run_once(), + "seeded runs should dispatch identical payments" + ); +} + +/// An activity source that is not a controlled node fails validation with a specific error. +#[rstest] +#[ntest::timeout(120_000)] +fn unknown_source_rejected() { + let network = SimulatedNetwork::new(); + let error = run_expecting_error( + &network, + vec![json!({ + "source": "ghost", + "destination": network.nodes()[1].alias, + "interval_secs": 2, + "amount_msat": 1000, + })], + ); + assert!( + error.contains("not found in nodes"), + "expected unknown-source validation error, got: {error}" + ); +} + +/// An activity destination that exists nowhere in the graph fails validation. +#[rstest] +#[ntest::timeout(120_000)] +fn unknown_destination_rejected() { + let network = SimulatedNetwork::new(); + let secp = Secp256k1::new(); + let stranger = PublicKey::from_secret_key(&secp, &SecretKey::from_slice(&[99u8; 32]).unwrap()); + + let error = run_expecting_error( + &network, + vec![json!({ + "source": network.nodes()[0].alias, + "destination": stranger.to_string(), + "interval_secs": 2, + "amount_msat": 1000, + })], + ); + assert!( + error.contains("unknown activity destination"), + "expected unknown-destination validation error, got: {error}" + ); +} + +/// A network where two nodes share an alias is rejected, since aliases would be ambiguous +/// references. +#[rstest] +#[ntest::timeout(120_000)] +fn duplicate_alias_rejected() { + let network = SimulatedNetwork::with_duplicate_alias(); + let nodes = network.nodes(); + let error = run_expecting_error( + &network, + vec![json!({ + "source": nodes[0].pubkey.to_string(), + "destination": nodes[1].pubkey.to_string(), + "interval_secs": 2, + "amount_msat": 1000, + })], + ); + assert!( + error.contains("duplicated alias"), + "expected duplicate-alias validation error, got: {error}" + ); +} + +/// The untagged `nodes` section infers the right connector implementation purely from which +/// fields each entry carries. Parsing only; no connections are attempted. +#[rstest] +#[ntest::timeout(120_000)] +fn node_connection_implementation_inference() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("sim.json"); + std::fs::write( + &path, + json!({ + "nodes": [ + { + "id": "lnd-node", + "address": "localhost:10009", + "macaroon": "/tmp/simln-test/admin.macaroon", + "cert": "/tmp/simln-test/tls.cert", + }, + { + "id": "cln-node", + "address": "localhost:9736", + "ca_cert": "/tmp/simln-test/ca.pem", + "client_cert": "/tmp/simln-test/client.pem", + "client_key": "/tmp/simln-test/client-key.pem", + }, + { + "id": "eclair-node", + "base_url": "127.0.0.1:8080", + "api_username": "", + "api_password": "eclair-password", + }, + { + "address": "https://127.0.0.1:3000", + "api_key": "6d6b3f5e", + "cert": "/tmp/simln-test/ldk-tls.crt", + }, + ], + }) + .to_string(), + ) + .expect("write sim file"); + + let params = parse_params(&path).expect("nodes section should deserialize"); + assert_eq!(params.nodes.len(), 4); + assert!(matches!(params.nodes[0], NodeConnection::Lnd(_))); + assert!(matches!(params.nodes[1], NodeConnection::Cln(_))); + assert!(matches!(params.nodes[2], NodeConnection::Eclair(_))); + assert!(matches!(params.nodes[3], NodeConnection::LdkServer(_))); +}