test(harness): sweep hardcoded binary paths + unguarded child spawns

CHANGELOG.md

@@ -27,6 +27,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   off-loop fsync agent).
 
 ### Fixed
+- **Test-harness hygiene sweep: hardcoded `target/release/moon` paths and
+  unguarded child spawns across `tests/`.** 33 integration suites resolved
+  the moon server binary via a bare `./target/release/moon` default, a
+  `CARGO_MANIFEST_DIR`-relative guess, or a local `find_moon_binary()` copy
+  that never checked `CARGO_BIN_EXE_moon` — a stale binary of unknown
+  provenance on a shared checkout could silently run instead of the one
+  cargo actually built for the test; migrated to `common::find_moon_binary()`
+  (5 suites with a documented "skip gracefully when unbuilt" contract keep
+  their own `Option<PathBuf>` resolver, with the same `CARGO_BIN_EXE_moon`
+  tier added ahead of the stale-path fallback). Separately, 6 suites
+  (`console_gateway_test`, `scan_fanout_multishard`,
+  `allocator_mimalloc_smoke`, `perf_v0112_arenas_cap`,
+  `replication_readonly_eval`, `replication_readonly_ws_mq`) held a bare
+  `Child` across many `assert!`/`.expect()` calls with cleanup only at the
+  end of the function — a mid-test panic orphaned the server, the same
+  shape behind issue #366's 667%-CPU incident — and now use the
+  kill-on-drop `MoonGuard` pattern from `tests/bgsave_startup_race.rs`.
+  Complex multi-restart harnesses (crash-recovery kill-9 cycles, Jepsen,
+  instance-lock, SIGTERM) were left untouched by design.
 - **`tests/bgsave_startup_race.rs` can no longer orphan its server on a
   mid-test panic** — the spawned child is now held by a kill-on-drop guard
   (the pattern from `tests/dir_deleted_degraded.rs`). An orphan from this

tests/admin_auth_cors_ratelimit.rs

@@ -43,7 +43,12 @@ impl Drop for Moon {
 }
 
 fn bin_path() -> std::path::PathBuf {
-    std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("target/release/moon")
+    // MOON_BIN -> CARGO_BIN_EXE_moon (cargo guarantees this exists for any
+    // test binary that references it) -> target/{release,debug} — see
+    // `common::find_moon_binary`. The bare `target/release/moon` guess this
+    // used to return unconditionally could silently pick a stale binary of
+    // unknown provenance on a shared checkout.
+    common::find_moon_binary()
 }
 
 /// GET `url` with the given headers, returning the raw response even for

tests/allocator_mimalloc_smoke.rs

@@ -62,14 +62,30 @@ fn wait_ready(port: u16) {
     panic!("moon did not become ready on port {port}");
 }
 
+/// Kill-on-drop guard: `wait_ready` and the redis-cli `.expect()` calls below
+/// can all panic before the old manual `child.kill()` ran, orphaning the
+/// server (task: test/harness-hygiene-sweep). See
+/// tests/bgsave_startup_race.rs for the same pattern.
+struct MoonGuard(Option<std::process::Child>);
+
+impl Drop for MoonGuard {
+    fn drop(&mut self) {
+        if let Some(mut child) = self.0.take() {
+            let _ = child.kill();
+            let _ = child.wait();
+        }
+    }
+}
+
 #[test]
 fn mimalloc_set_get_roundtrip() {
     if !redis_cli_available() {
         eprintln!("skipping: redis-cli not on PATH");
         return;
     }
     let port = 16401;
-    let mut child = spawn_moon(port);
+    let child = spawn_moon(port);
+    let child = MoonGuard(Some(child));
     wait_ready(port);
 
     let set_out = Command::new("redis-cli")
@@ -84,8 +100,7 @@ fn mimalloc_set_get_roundtrip() {
         .expect("GET failed to spawn");
     let get_stdout = String::from_utf8_lossy(&get_out.stdout);
 
-    let _ = child.kill();
-    let _ = child.wait();
+    drop(child); // MoonGuard SIGKILLs + reaps
 
     assert!(
         set_stdout.contains("OK"),

tests/aof_fsync_err_subscribe_ordering.rs

@@ -22,6 +22,8 @@
 //! Requires the release binary at ./target/release/moon.
 //! The MOON_TEST_AOF_FSYNC_FAIL=1 env var is passed to the child process.
 
+mod common;
+
 use std::io::{BufRead, BufReader, Write};
 use std::net::TcpStream;
 use std::path::PathBuf;
@@ -46,7 +48,7 @@ fn spawn_moon_with_fsync_fail(port: u16, shards: u16) -> (Child, PathBuf) {
     let stderr_log = dir.join("moon.stderr.log");
     let stdout_log = dir.join("moon.stdout.log");
 
-    let child = Command::new("./target/release/moon")
+    let child = Command::new(common::find_moon_binary())
         .args([
             "--port",
             &port.to_string(),

tests/aof_multidb_kill9.rs

@@ -22,13 +22,20 @@
 
 #![cfg(any(feature = "runtime-monoio", feature = "runtime-tokio"))]
 
+mod common;
+
 use std::io::{BufRead, BufReader, Write};
 use std::net::TcpStream;
 use std::process::{Child, Command, Stdio};
 use std::time::Duration;
 
-fn moon_bin() -> String {
-    std::env::var("MOON_BIN").unwrap_or_else(|_| "./target/release/moon".to_string())
+fn moon_bin() -> std::path::PathBuf {
+    // Migrated off the bare `./target/release/moon` default (task:
+    // test/harness-hygiene-sweep) — a stale target/release/moon of unknown
+    // provenance would silently run the wrong binary. See
+    // `common::find_moon_binary` for the MOON_BIN -> CARGO_BIN_EXE_moon ->
+    // target/{release,debug} precedence.
+    common::find_moon_binary()
 }
 
 fn start_moon(port: u16, dir: &str, shards: usize) -> Child {

tests/aof_toplevel_multishard_refusal.rs

@@ -13,6 +13,8 @@
 //!
 //! Requires the release binary at ./target/release/moon.
 
+mod common;
+
 use std::fs;
 use std::io::Read as _;
 use std::path::PathBuf;
@@ -70,7 +72,7 @@ fn toplevel_manifest_with_multishard_exits_2_and_prints_refusing_to_start() {
     let stderr_log = dir.join("moon.stderr.log");
     let stdout_log = dir.join("moon.stdout.log");
 
-    let mut child = Command::new("./target/release/moon")
+    let mut child = Command::new(common::find_moon_binary())
         .args([
             "--port",
             "17399", // high port unlikely to clash
@@ -156,7 +158,7 @@ fn toplevel_manifest_with_single_shard_is_allowed() {
     let stderr_log = dir.join("moon.stderr.log");
     let stdout_log = dir.join("moon.stdout.log");
 
-    let mut child = Command::new("./target/release/moon")
+    let mut child = Command::new(common::find_moon_binary())
         .args([
             "--port",
             "17400",

tests/bgsave_startup_race.rs

@@ -94,7 +94,7 @@ fn bgsave_issued_during_shard_startup_still_completes() {
     // guard instead of orphaning the server. An orphan from THIS suite is
     // what produced the 667%-CPU incident behind issue #366 (leaked child,
     // tmpdir later cleaned, pre-fix binary error-looping its tick).
-    let mut child = MoonGuard(Some(child));
+    let child = MoonGuard(Some(child));
 
     let mut c = Conn::open(port);
     // PING to confirm command plane is up, then fire BGSAVE immediately —

tests/cold_shadow_overwrite_resurrection.rs

@@ -46,6 +46,8 @@
 
 #![cfg(any(feature = "runtime-monoio", feature = "runtime-tokio"))]
 
+mod common;
+
 use std::io::Write;
 use std::process::{Child, Command, Stdio};
 use std::time::Duration;
@@ -95,7 +97,7 @@ fn unique_dir(suffix: &str) -> std::path::PathBuf {
 fn start_moon(port: u16, dir: &std::path::Path) -> Child {
     let off_dir = dir.join("off");
     std::fs::create_dir_all(&off_dir).expect("create off dir");
-    Command::new("./target/release/moon")
+    Command::new(common::find_moon_binary())
         .args([
             "--port",
             &port.to_string(),

tests/console_gateway_test.rs

@@ -7,11 +7,13 @@
 
 #![cfg(feature = "console")]
 
+mod common;
+
 use std::process::{Child, Command};
 use std::time::Duration;
 
 fn start_server() -> Child {
-    Command::new("./target/release/moon")
+    Command::new(common::find_moon_binary())
         .args(["--port", "16399", "--admin-port", "16400", "--shards", "2"])
         .spawn()
         .expect("Failed to start moon server (build with --release --features console,graph first)")
@@ -44,10 +46,26 @@ fn curl_json(method: &str, url: &str, body: Option<&str>) -> (u16, String) {
     (status, resp_body)
 }
 
+/// Kill-on-drop guard: every assert! below panics through the guard instead
+/// of orphaning the server (task: test/harness-hygiene-sweep — a leaked
+/// child from a suite exactly this shape produced the 667%-CPU incident
+/// behind issue #366). See tests/bgsave_startup_race.rs for the same
+/// pattern.
+struct MoonGuard(Option<Child>);
+
+impl Drop for MoonGuard {
+    fn drop(&mut self) {
+        if let Some(mut child) = self.0.take() {
+            common::sigkill(&mut child);
+        }
+    }
+}
+
 #[test]
 #[ignore] // Requires a built release binary; run explicitly
 fn test_rest_api_endpoints() {
-    let mut server = start_server();
+    let server = start_server();
+    let server = MoonGuard(Some(server));
     wait_for_server(16400);
 
     // 1. POST /api/v1/command -- SET a key
@@ -135,7 +153,7 @@ fn test_rest_api_endpoints() {
         body
     );
 
-    // Cleanup
-    server.kill().ok();
-    server.wait().ok();
+    // MoonGuard SIGKILLs + reaps on drop (end of scope, or on an earlier
+    // panic from any assert! above).
+    drop(server);
 }

tests/crash_matrix_per_shard_aof.rs

@@ -33,6 +33,8 @@
 
 #![cfg(any(feature = "runtime-monoio", feature = "runtime-tokio"))]
 
+mod common;
+
 use std::process::{Child, Command, Stdio};
 use std::time::Duration;
 
@@ -85,7 +87,7 @@ fn start_moon(port: u16, dir: &std::path::Path) -> Child {
 }
 
 fn start_moon_with_fsync(port: u16, dir: &std::path::Path, fsync: &str) -> Child {
-    Command::new("./target/release/moon")
+    Command::new(common::find_moon_binary())
         .args([
             "--port",
             &port.to_string(),

tests/crash_matrix_per_shard_bgrewriteaof.rs

@@ -26,6 +26,8 @@
 
 #![cfg(any(feature = "runtime-monoio", feature = "runtime-tokio"))]
 
+mod common;
+
 use std::process::{Child, Command, Stdio};
 use std::time::Duration;
 
@@ -51,7 +53,7 @@ fn unique_dir(suffix: &str) -> std::path::PathBuf {
 }
 
 fn start_moon(port: u16, dir: &std::path::Path) -> Child {
-    Command::new("./target/release/moon")
+    Command::new(common::find_moon_binary())
         .args([
             "--port",
             &port.to_string(),

tests/crash_recovery_cold_del_resurrection.rs

@@ -37,6 +37,8 @@
 
 #![cfg(any(feature = "runtime-monoio", feature = "runtime-tokio"))]
 
+mod common;
+
 use std::io::Write;
 use std::process::{Child, Command, Stdio};
 use std::time::Duration;
@@ -83,7 +85,7 @@ fn unique_dir(suffix: &str) -> std::path::PathBuf {
 fn start_moon(port: u16, dir: &std::path::Path) -> Child {
     let off_dir = dir.join("off");
     std::fs::create_dir_all(&off_dir).expect("create off dir");
-    Command::new("./target/release/moon")
+    Command::new(common::find_moon_binary())
         .args([
             "--port",
             &port.to_string(),

tests/crash_recovery_disk_offload_no_aof.rs

@@ -83,6 +83,8 @@
 
 #![cfg(any(feature = "runtime-monoio", feature = "runtime-tokio"))]
 
+mod common;
+
 use std::collections::HashSet;
 use std::io::Write;
 use std::process::{Child, Command, Stdio};
@@ -134,7 +136,7 @@ fn unique_dir(suffix: &str) -> std::path::PathBuf {
 fn start_moon(port: u16, dir: &std::path::Path) -> Child {
     let off_dir = dir.join("off");
     std::fs::create_dir_all(&off_dir).expect("create off dir");
-    Command::new("./target/release/moon")
+    Command::new(common::find_moon_binary())
         .args([
             "--port",
             &port.to_string(),

tests/crash_recovery_graph_durability.rs

@@ -36,6 +36,8 @@
 #![cfg(any(feature = "runtime-monoio", feature = "runtime-tokio"))]
 #![allow(clippy::unwrap_used)]
 
+mod common;
+
 use std::collections::BTreeSet;
 use std::io::{BufReader, Read, Write};
 use std::net::{TcpStream, ToSocketAddrs};
@@ -44,29 +46,14 @@ use std::process::{Child, Command};
 use std::time::{Duration, Instant};
 
 // ---------------------------------------------------------------------------
-// Binary resolution (pattern: tests/crash_recovery_vector_durability.rs)
+// Binary resolution — delegates to common::find_moon_binary (task:
+// test/harness-hygiene-sweep). The old local copy stopped at
+// target/{release,debug}/moon and never checked CARGO_BIN_EXE_moon, the
+// binary cargo actually built for THIS test run.
 // ---------------------------------------------------------------------------
 
 fn find_moon_binary() -> PathBuf {
-    if let Ok(bin) = std::env::var("MOON_BIN") {
-        let p = PathBuf::from(bin);
-        if p.exists() {
-            return p;
-        }
-    }
-    let manifest = env!("CARGO_MANIFEST_DIR");
-    let release = PathBuf::from(format!("{manifest}/target/release/moon"));
-    if release.exists() {
-        return release;
-    }
-    let debug = PathBuf::from(format!("{manifest}/target/debug/moon"));
-    if debug.exists() {
-        return debug;
-    }
-    panic!(
-        "No moon binary found. Build with `cargo build --release` or set \
-         MOON_BIN=/path/to/moon."
-    );
+    common::find_moon_binary()
 }
 
 fn unique_port() -> u16 {

tests/crash_recovery_orphan_sweep_readiness.rs

@@ -38,6 +38,8 @@
 
 #![cfg(any(feature = "runtime-monoio", feature = "runtime-tokio"))]
 
+mod common;
+
 use std::process::{Child, Command, Stdio};
 use std::time::{Duration, Instant};
 
@@ -78,7 +80,7 @@ fn unique_dir(suffix: &str) -> std::path::PathBuf {
 }
 
 fn start_moon(port: u16, dir: &std::path::Path, off_dir: &std::path::Path) -> Child {
-    Command::new("./target/release/moon")
+    Command::new(common::find_moon_binary())
         .args([
             "--port",
             &port.to_string(),