Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 56 additions & 10 deletions crates/lance-context-wal/src/replication/runtime/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -698,6 +698,28 @@ struct NodePools {
control: Pool,
}

/// Kept outside the timed future so cancellation preserves its last boundary.
/// Updating this state does not allocate or expose addresses and request values.
#[derive(Default)]
struct ExchangeProgress {
rpc: Option<(&'static str, u64, u64)>,
stage: &'static str,
reused: Option<bool>,
}

impl std::fmt::Display for ExchangeProgress {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let Some((operation, group, node)) = self.rpc else {
return f.write_str("WAL RPC not started");
};
write!(f, "WAL RPC {operation} group={group} node={node}")?;
if let Some(reused) = self.reused {
write!(f, " reused={reused}")?;
}
write!(f, " stage={}", self.stage)
}
}

fn disconnected(error: &anyhow::Error) -> bool {
use std::io::ErrorKind;
error.downcast_ref::<std::io::Error>().is_some_and(|e| {
Expand Down Expand Up @@ -839,9 +861,13 @@ impl Client {
/// retried. A read on a disconnected pooled stream may reconnect once within
/// the original deadline; a cached leader hint is never treated as an ACK.
pub async fn request(&self, node: u64, request: Request) -> Result<Response> {
timeout(self.options.timeout, self.exchange(node, request))
.await
.context("client exchange timed out; outcome unknown")?
let mut progress = ExchangeProgress::default();
timeout(
self.options.timeout,
self.exchange(node, request, &mut progress),
)
.await
.with_context(|| format!("client exchange timed out; {progress}; outcome unknown"))?
}
/// Follow configured-voter hints only when the remote confirms it did not
/// submit the request. Unknown mutation outcomes require explicit exact retry.
Expand All @@ -853,6 +879,7 @@ impl Client {
/// method's single overall deadline. Queries, source reads and payload fetches
/// can move to another configured voter after a disconnect or refused connection.
pub async fn request_routed(&self, request: Request) -> Result<Response> {
let mut progress = ExchangeProgress::default();
timeout(self.options.timeout, async {
let group = match &request {
Request::Append { group, .. }
Expand All @@ -873,7 +900,7 @@ impl Client {
let mut visited = BTreeSet::new();
loop {
visited.insert(node);
let response = match self.exchange(node, request.clone()).await {
let response = match self.exchange(node, request.clone(), &mut progress).await {
Ok(response) => response,
Err(error) => {
let next = voters[(voters.iter().position(|id| *id == node).unwrap() + 1)
Expand All @@ -895,6 +922,7 @@ impl Client {
node = next;
if visited.len() == voters.len() {
visited.clear();
progress.stage = "routing_backoff";
tokio::time::sleep(Duration::from_millis(20)).await;
}
continue;
Expand All @@ -912,6 +940,7 @@ impl Client {
// Every voter explicitly declined before execution.
// Release reply credits and wait before refreshing the
// hints; a transient election is not a protocol fault.
progress.stage = "routing_backoff";
tokio::time::sleep(Duration::from_millis(20)).await;
visited.clear();
if voters.contains(&leader) {
Expand Down Expand Up @@ -952,6 +981,7 @@ impl Client {
drop(response);
node = next;
visited.clear();
progress.stage = "routing_backoff";
tokio::time::sleep(Duration::from_millis(20)).await;
continue;
}
Expand All @@ -966,9 +996,14 @@ impl Client {
}
})
.await
.context("routed exchange timed out; outcome unknown")?
.with_context(|| format!("routed exchange timed out; {progress}; outcome unknown"))?
}
async fn exchange(&self, node: u64, request: Request) -> Result<Response> {
async fn exchange(
&self,
node: u64,
request: Request,
progress: &mut ExchangeProgress,
) -> Result<Response> {
let pools = self.nodes.get(&node).context("unknown client target")?;
let (wire, body) = match request {
Request::Append { group, batch } => {
Expand Down Expand Up @@ -999,6 +1034,11 @@ impl Client {
};
let data = wire.data();
let (group, operation) = wire.diagnostic(self.catalog);
*progress = ExchangeProgress {
rpc: Some((operation, group, node)),
stage: "admission",
reused: None,
};
let read_only = matches!(
wire,
WireRequest::Query { .. }
Expand Down Expand Up @@ -1048,19 +1088,22 @@ impl Client {
// context only on failure; addresses, credentials and payloads stay out.
let connection_error =
|stage| format!("WAL RPC {operation} group={group} node={node} stage={stage}");
progress.stage = "pool_admission";
let _slot = pool
.slots
.clone()
.acquire_owned()
.await
.with_context(|| connection_error("pool_admission"))?;
progress.stage = "idle_connection";
let mut idle = pool
.take_idle()
.await
.with_context(|| connection_error("idle_connection"))?;
let mut reconnect_read = read_only && idle.is_some();
loop {
let reused = idle.is_some();
progress.reused = Some(reused);
let mut stream = match idle.take() {
Some(stream) => stream,
None => {
Expand All @@ -1069,10 +1112,12 @@ impl Client {
} else {
&pools.endpoint.control_address
};
progress.stage = "tcp_connect";
let tcp = TcpStream::connect(address)
.await
.with_context(|| connection_error("tcp_connect"))?;
tcp.set_nodelay(true)?;
progress.stage = "tls_handshake";
let stream = self
.connector
.connect(
Expand All @@ -1091,7 +1136,7 @@ impl Client {
}
};
drop(connection.take());
let mut stage = "send";
progress.stage = "send";
let response: Result<(Reply, Bytes)> = async {
let sending = timings.map(|c| c[2].start());
let id = self
Expand All @@ -1101,15 +1146,15 @@ impl Client {
send(&mut stream, id, &metadata, &body).await?;
drop(sending);
let _receiving = timings.map(|c| c[3].start());
stage = "response_prefix";
progress.stage = "response_prefix";
let (received_id, meta_len, body_len) =
prefix(&mut stream, if fetch { self.max_record } else { 0 }).await?;
ensure!(received_id == id, "client response correlation mismatch");
ensure!(
meta_len <= reply_meta,
"client response metadata exceeds request bound"
);
stage = "response_metadata";
progress.stage = "response_metadata";
let metadata = receive(&mut stream, meta_len).await?;
let (observed, reply): (Scope, Reply) = codec().deserialize(&metadata)?;
ensure!(observed == expected, "foreign client response scope");
Expand Down Expand Up @@ -1141,7 +1186,7 @@ impl Client {
Reply::Record(_) => anyhow::bail!("record reply for non-fetch request"),
_ => ensure!(body_len == 0 && !fetch, "unexpected response payload/type"),
}
stage = "response_body";
progress.stage = "response_body";
let body = receive(&mut stream, body_len).await?;
Ok((reply, body))
}
Expand All @@ -1159,6 +1204,7 @@ impl Client {
Err(error) => {
// A transport stage does not establish whether a mutation
// committed. Preserve the cause and original retry policy.
let stage = progress.stage;
return Err(error.context(format!(
"WAL RPC {operation} group={group} node={node} reused={reused} stage={stage}; outcome unknown"
)));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ mod routing;
mod routing_hint;
mod routing_transport;
mod source_read;
mod timeout_diagnostics;

struct RpcCluster {
net: Cluster,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,15 @@ async fn publisher_binding_response_timeout_finishes_timing() -> Result<()> {
arrived.await?;
let expired = c.client.request(leader, binding_request()).await;
drop(release);
assert!(expired.is_err());
let error = expired
.err()
.context("held response unexpectedly completed")?;
let message = format!("{error:#}");
assert!(message.contains("stage=response_prefix"), "{message}");
assert!(message.contains("reused=true"), "{message}");
assert!(error
.downcast_ref::<tokio::time::error::Elapsed>()
.is_some());
let response = &c.client.publisher_binding_timings()["response"];
assert_eq!(response.started, 2);
assert_eq!(response.finished, 2);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
use super::*;

fn binding() -> Request {
Request::Query {
group: 10,
query: query::Query::PublisherBinding {
table: uuid::Uuid::from_u128(123),
},
}
}

fn assert_deadline(error: &anyhow::Error, node: u64, stage: &str, routed: bool) {
let message = format!("{error:#}");
let kind = if routed { "routed" } else { "client" };
assert!(
message.contains(&format!("{kind} exchange timed out")),
"{message}"
);
assert!(
message.contains(&format!(
"WAL RPC query.publisher_binding group=10 node={node}"
)),
"{message}"
);
assert!(message.contains(&format!("stage={stage}")), "{message}");
assert!(message.contains("outcome unknown"), "{message}");
assert!(error
.downcast_ref::<tokio::time::error::Elapsed>()
.is_some());
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn timeout_diagnostics_distinguish_byte_and_pool_admission() -> Result<()> {
let dir = tempfile::tempdir()?;
let c = RpcCluster::open_with_options(
dir.path(),
Options::default(),
Options {
connections: 1,
timeout: Duration::from_millis(300),
..Options::default()
},
)
.await?;
let leader = c.leader().await?;
for routed in [false, true] {
for (semaphore, stage) in [
(&c.client.budgets.control, "admission"),
(&c.client.nodes[&leader].control.slots, "pool_admission"),
] {
let held = semaphore
.clone()
.acquire_many_owned(semaphore.available_permits() as u32)
.await?;
c.client.leaders.lock().unwrap().insert(10, leader);
let result = if routed {
c.client.request_routed(binding()).await
} else {
c.client.request(leader, binding()).await
};
drop(held);
let error = result.err().context("blocked RPC unexpectedly completed")?;
assert_deadline(&error, leader, stage, routed);
assert_eq!(c.client.publisher_binding_timings()["send"].started, 0);
assert_eq!(
c.client.budgets.control.available_permits(),
c.client.options.control_bytes
);
assert_eq!(c.client.nodes[&leader].control.slots.available_permits(), 1);
}
}
c.shutdown().await
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn timeout_diagnostics_identify_held_tls_without_replaying() -> Result<()> {
let dir = tempfile::tempdir()?;
let c = RpcCluster::open(dir.path(), Options::default()).await?;
let leader = c.leader().await?;
let listener = Arc::new(TcpListener::bind("127.0.0.1:0").await?);
let address = listener.local_addr()?.to_string();
let mut endpoints = c.endpoints.clone();
endpoints.get_mut(&leader).unwrap().control_address = address.clone();
let mut tls = (*c.net.tls[&4].client).clone();
tls.alpn_protocols = vec![ALPN.to_vec()];
let client = Client::new(
&layout(),
&Limits::default(),
endpoints,
Arc::new(tls),
Options {
connections: 1,
timeout: Duration::from_millis(300),
..Options::default()
},
)?;
for routed in [false, true] {
let listener = listener.clone();
let (entered, arrived) = tokio::sync::oneshot::channel();
let (release, resumed) = tokio::sync::oneshot::channel();
// Accept TCP but deliberately withhold the TLS handshake. This peer
// never receives an application request or simulates a durable result.
let server = tokio::spawn(async move {
let (tcp, _) = listener.accept().await?;
let _ = entered.send(());
resumed.await?;
drop(tcp);
Ok::<_, anyhow::Error>(())
});
client.leaders.lock().unwrap().insert(10, leader);
let request = async {
if routed {
client.request_routed(binding()).await
} else {
client.request(leader, binding()).await
}
};
tokio::pin!(request);
tokio::select! {
result = &mut request => anyhow::bail!("RPC ended before TCP accept: {}", result.is_ok()),
result = arrived => result?,
}
let error = request
.await
.err()
.context("held TLS unexpectedly completed")?;
release.send(()).unwrap();
server.await??;
assert_deadline(&error, leader, "tls_handshake", routed);
assert!(!format!("{error:#}").contains(&address));
assert_eq!(client.publisher_binding_timings()["send"].started, 0);
assert_eq!(
client.budgets.control.available_permits(),
client.options.control_bytes
);
assert_eq!(client.nodes[&leader].control.slots.available_permits(), 1);
}
assert!(timeout(Duration::from_millis(30), listener.accept())
.await
.is_err());
c.shutdown().await
}
Loading