diff --git a/crates/lance-context-wal/src/replication/runtime/client.rs b/crates/lance-context-wal/src/replication/runtime/client.rs index 8b1d9afa..0c9ba83c 100644 --- a/crates/lance-context-wal/src/replication/runtime/client.rs +++ b/crates/lance-context-wal/src/replication/runtime/client.rs @@ -698,6 +698,28 @@ struct NodePools { control: Pool, } +/// Kept outside the timed future so cancellation preserves its last boundary. +/// Updating this state does not allocate or expose addresses and request values. +#[derive(Default)] +struct ExchangeProgress { + rpc: Option<(&'static str, u64, u64)>, + stage: &'static str, + reused: Option, +} + +impl std::fmt::Display for ExchangeProgress { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Some((operation, group, node)) = self.rpc else { + return f.write_str("WAL RPC not started"); + }; + write!(f, "WAL RPC {operation} group={group} node={node}")?; + if let Some(reused) = self.reused { + write!(f, " reused={reused}")?; + } + write!(f, " stage={}", self.stage) + } +} + fn disconnected(error: &anyhow::Error) -> bool { use std::io::ErrorKind; error.downcast_ref::().is_some_and(|e| { @@ -839,9 +861,13 @@ impl Client { /// retried. A read on a disconnected pooled stream may reconnect once within /// the original deadline; a cached leader hint is never treated as an ACK. pub async fn request(&self, node: u64, request: Request) -> Result { - timeout(self.options.timeout, self.exchange(node, request)) - .await - .context("client exchange timed out; outcome unknown")? + let mut progress = ExchangeProgress::default(); + timeout( + self.options.timeout, + self.exchange(node, request, &mut progress), + ) + .await + .with_context(|| format!("client exchange timed out; {progress}; outcome unknown"))? } /// Follow configured-voter hints only when the remote confirms it did not /// submit the request. Unknown mutation outcomes require explicit exact retry. @@ -853,6 +879,7 @@ impl Client { /// method's single overall deadline. Queries, source reads and payload fetches /// can move to another configured voter after a disconnect or refused connection. pub async fn request_routed(&self, request: Request) -> Result { + let mut progress = ExchangeProgress::default(); timeout(self.options.timeout, async { let group = match &request { Request::Append { group, .. } @@ -873,7 +900,7 @@ impl Client { let mut visited = BTreeSet::new(); loop { visited.insert(node); - let response = match self.exchange(node, request.clone()).await { + let response = match self.exchange(node, request.clone(), &mut progress).await { Ok(response) => response, Err(error) => { let next = voters[(voters.iter().position(|id| *id == node).unwrap() + 1) @@ -895,6 +922,7 @@ impl Client { node = next; if visited.len() == voters.len() { visited.clear(); + progress.stage = "routing_backoff"; tokio::time::sleep(Duration::from_millis(20)).await; } continue; @@ -912,6 +940,7 @@ impl Client { // Every voter explicitly declined before execution. // Release reply credits and wait before refreshing the // hints; a transient election is not a protocol fault. + progress.stage = "routing_backoff"; tokio::time::sleep(Duration::from_millis(20)).await; visited.clear(); if voters.contains(&leader) { @@ -952,6 +981,7 @@ impl Client { drop(response); node = next; visited.clear(); + progress.stage = "routing_backoff"; tokio::time::sleep(Duration::from_millis(20)).await; continue; } @@ -966,9 +996,14 @@ impl Client { } }) .await - .context("routed exchange timed out; outcome unknown")? + .with_context(|| format!("routed exchange timed out; {progress}; outcome unknown"))? } - async fn exchange(&self, node: u64, request: Request) -> Result { + async fn exchange( + &self, + node: u64, + request: Request, + progress: &mut ExchangeProgress, + ) -> Result { let pools = self.nodes.get(&node).context("unknown client target")?; let (wire, body) = match request { Request::Append { group, batch } => { @@ -999,6 +1034,11 @@ impl Client { }; let data = wire.data(); let (group, operation) = wire.diagnostic(self.catalog); + *progress = ExchangeProgress { + rpc: Some((operation, group, node)), + stage: "admission", + reused: None, + }; let read_only = matches!( wire, WireRequest::Query { .. } @@ -1048,12 +1088,14 @@ impl Client { // context only on failure; addresses, credentials and payloads stay out. let connection_error = |stage| format!("WAL RPC {operation} group={group} node={node} stage={stage}"); + progress.stage = "pool_admission"; let _slot = pool .slots .clone() .acquire_owned() .await .with_context(|| connection_error("pool_admission"))?; + progress.stage = "idle_connection"; let mut idle = pool .take_idle() .await @@ -1061,6 +1103,7 @@ impl Client { let mut reconnect_read = read_only && idle.is_some(); loop { let reused = idle.is_some(); + progress.reused = Some(reused); let mut stream = match idle.take() { Some(stream) => stream, None => { @@ -1069,10 +1112,12 @@ impl Client { } else { &pools.endpoint.control_address }; + progress.stage = "tcp_connect"; let tcp = TcpStream::connect(address) .await .with_context(|| connection_error("tcp_connect"))?; tcp.set_nodelay(true)?; + progress.stage = "tls_handshake"; let stream = self .connector .connect( @@ -1091,7 +1136,7 @@ impl Client { } }; drop(connection.take()); - let mut stage = "send"; + progress.stage = "send"; let response: Result<(Reply, Bytes)> = async { let sending = timings.map(|c| c[2].start()); let id = self @@ -1101,7 +1146,7 @@ impl Client { send(&mut stream, id, &metadata, &body).await?; drop(sending); let _receiving = timings.map(|c| c[3].start()); - stage = "response_prefix"; + progress.stage = "response_prefix"; let (received_id, meta_len, body_len) = prefix(&mut stream, if fetch { self.max_record } else { 0 }).await?; ensure!(received_id == id, "client response correlation mismatch"); @@ -1109,7 +1154,7 @@ impl Client { meta_len <= reply_meta, "client response metadata exceeds request bound" ); - stage = "response_metadata"; + progress.stage = "response_metadata"; let metadata = receive(&mut stream, meta_len).await?; let (observed, reply): (Scope, Reply) = codec().deserialize(&metadata)?; ensure!(observed == expected, "foreign client response scope"); @@ -1141,7 +1186,7 @@ impl Client { Reply::Record(_) => anyhow::bail!("record reply for non-fetch request"), _ => ensure!(body_len == 0 && !fetch, "unexpected response payload/type"), } - stage = "response_body"; + progress.stage = "response_body"; let body = receive(&mut stream, body_len).await?; Ok((reply, body)) } @@ -1159,6 +1204,7 @@ impl Client { Err(error) => { // A transport stage does not establish whether a mutation // committed. Preserve the cause and original retry policy. + let stage = progress.stage; return Err(error.context(format!( "WAL RPC {operation} group={group} node={node} reused={reused} stage={stage}; outcome unknown" ))); diff --git a/crates/lance-context-wal/src/replication/runtime/client/tests.rs b/crates/lance-context-wal/src/replication/runtime/client/tests.rs index 42fe8be9..034d5074 100644 --- a/crates/lance-context-wal/src/replication/runtime/client/tests.rs +++ b/crates/lance-context-wal/src/replication/runtime/client/tests.rs @@ -20,6 +20,7 @@ mod routing; mod routing_hint; mod routing_transport; mod source_read; +mod timeout_diagnostics; struct RpcCluster { net: Cluster, diff --git a/crates/lance-context-wal/src/replication/runtime/client/tests/binding_timing.rs b/crates/lance-context-wal/src/replication/runtime/client/tests/binding_timing.rs index 4ea28fac..396d7688 100644 --- a/crates/lance-context-wal/src/replication/runtime/client/tests/binding_timing.rs +++ b/crates/lance-context-wal/src/replication/runtime/client/tests/binding_timing.rs @@ -134,7 +134,15 @@ async fn publisher_binding_response_timeout_finishes_timing() -> Result<()> { arrived.await?; let expired = c.client.request(leader, binding_request()).await; drop(release); - assert!(expired.is_err()); + let error = expired + .err() + .context("held response unexpectedly completed")?; + let message = format!("{error:#}"); + assert!(message.contains("stage=response_prefix"), "{message}"); + assert!(message.contains("reused=true"), "{message}"); + assert!(error + .downcast_ref::() + .is_some()); let response = &c.client.publisher_binding_timings()["response"]; assert_eq!(response.started, 2); assert_eq!(response.finished, 2); diff --git a/crates/lance-context-wal/src/replication/runtime/client/tests/timeout_diagnostics.rs b/crates/lance-context-wal/src/replication/runtime/client/tests/timeout_diagnostics.rs new file mode 100644 index 00000000..dc33aada --- /dev/null +++ b/crates/lance-context-wal/src/replication/runtime/client/tests/timeout_diagnostics.rs @@ -0,0 +1,142 @@ +use super::*; + +fn binding() -> Request { + Request::Query { + group: 10, + query: query::Query::PublisherBinding { + table: uuid::Uuid::from_u128(123), + }, + } +} + +fn assert_deadline(error: &anyhow::Error, node: u64, stage: &str, routed: bool) { + let message = format!("{error:#}"); + let kind = if routed { "routed" } else { "client" }; + assert!( + message.contains(&format!("{kind} exchange timed out")), + "{message}" + ); + assert!( + message.contains(&format!( + "WAL RPC query.publisher_binding group=10 node={node}" + )), + "{message}" + ); + assert!(message.contains(&format!("stage={stage}")), "{message}"); + assert!(message.contains("outcome unknown"), "{message}"); + assert!(error + .downcast_ref::() + .is_some()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn timeout_diagnostics_distinguish_byte_and_pool_admission() -> Result<()> { + let dir = tempfile::tempdir()?; + let c = RpcCluster::open_with_options( + dir.path(), + Options::default(), + Options { + connections: 1, + timeout: Duration::from_millis(300), + ..Options::default() + }, + ) + .await?; + let leader = c.leader().await?; + for routed in [false, true] { + for (semaphore, stage) in [ + (&c.client.budgets.control, "admission"), + (&c.client.nodes[&leader].control.slots, "pool_admission"), + ] { + let held = semaphore + .clone() + .acquire_many_owned(semaphore.available_permits() as u32) + .await?; + c.client.leaders.lock().unwrap().insert(10, leader); + let result = if routed { + c.client.request_routed(binding()).await + } else { + c.client.request(leader, binding()).await + }; + drop(held); + let error = result.err().context("blocked RPC unexpectedly completed")?; + assert_deadline(&error, leader, stage, routed); + assert_eq!(c.client.publisher_binding_timings()["send"].started, 0); + assert_eq!( + c.client.budgets.control.available_permits(), + c.client.options.control_bytes + ); + assert_eq!(c.client.nodes[&leader].control.slots.available_permits(), 1); + } + } + c.shutdown().await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn timeout_diagnostics_identify_held_tls_without_replaying() -> Result<()> { + let dir = tempfile::tempdir()?; + let c = RpcCluster::open(dir.path(), Options::default()).await?; + let leader = c.leader().await?; + let listener = Arc::new(TcpListener::bind("127.0.0.1:0").await?); + let address = listener.local_addr()?.to_string(); + let mut endpoints = c.endpoints.clone(); + endpoints.get_mut(&leader).unwrap().control_address = address.clone(); + let mut tls = (*c.net.tls[&4].client).clone(); + tls.alpn_protocols = vec![ALPN.to_vec()]; + let client = Client::new( + &layout(), + &Limits::default(), + endpoints, + Arc::new(tls), + Options { + connections: 1, + timeout: Duration::from_millis(300), + ..Options::default() + }, + )?; + for routed in [false, true] { + let listener = listener.clone(); + let (entered, arrived) = tokio::sync::oneshot::channel(); + let (release, resumed) = tokio::sync::oneshot::channel(); + // Accept TCP but deliberately withhold the TLS handshake. This peer + // never receives an application request or simulates a durable result. + let server = tokio::spawn(async move { + let (tcp, _) = listener.accept().await?; + let _ = entered.send(()); + resumed.await?; + drop(tcp); + Ok::<_, anyhow::Error>(()) + }); + client.leaders.lock().unwrap().insert(10, leader); + let request = async { + if routed { + client.request_routed(binding()).await + } else { + client.request(leader, binding()).await + } + }; + tokio::pin!(request); + tokio::select! { + result = &mut request => anyhow::bail!("RPC ended before TCP accept: {}", result.is_ok()), + result = arrived => result?, + } + let error = request + .await + .err() + .context("held TLS unexpectedly completed")?; + release.send(()).unwrap(); + server.await??; + assert_deadline(&error, leader, "tls_handshake", routed); + assert!(!format!("{error:#}").contains(&address)); + assert_eq!(client.publisher_binding_timings()["send"].started, 0); + assert_eq!( + client.budgets.control.available_permits(), + client.options.control_bytes + ); + assert_eq!(client.nodes[&leader].control.slots.available_permits(), 1); + } + assert!(timeout(Duration::from_millis(30), listener.accept()) + .await + .is_err()); + c.shutdown().await +}