From 6e604cfe125ec51d811600fdb041b864186fbed0 Mon Sep 17 00:00:00 2001 From: Ilexpwh <122920329+Ilexpwh@users.noreply.github.com> Date: Sun, 30 Aug 2026 14:25:30 +0800 Subject: [PATCH 1/6] feat(server): add composable ingress hooks Signed-off-by: Ilexpwh <122920329+Ilexpwh@users.noreply.github.com> --- crates/switchyard-server/README.md | 50 ++++++ crates/switchyard-server/src/lib.rs | 210 ++++++++++++++++++----- crates/switchyard-server/tests/server.rs | 95 +++++++++- 3 files changed, 310 insertions(+), 45 deletions(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 2ff1a7a5a..fdf0e13c8 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -62,6 +62,56 @@ switchyard-server --config routes.toml Ctrl+C and Unix `SIGTERM` stop new connections and allow active requests to drain for up to `--shutdown-timeout` (30 seconds by default) before they are terminated. +## Embedding the LLM routes + +Rust hosts can install [`IngressHooks`] on [`ServerState`] and use [`build_llm_router`] to expose +only Chat Completions, Responses, and Messages. `prepare` runs before JSON body extraction, so a +host can authenticate or reject without buffering an untrusted body. `resolve_model` runs after +wire-format decoding and can map a public model to an internal route. `present_response` can adapt +the final error envelope. All methods have defaults that preserve the standalone server behavior. + +```rust +use std::sync::Arc; + +use async_trait::async_trait; +use axum::extract::Request as HttpRequest; +use axum::response::Response; +use switchyard_protocol::{Metadata, ModelId, Request, WireFormat}; +use switchyard_server::{IngressHooks, ServerState, build_llm_router}; + +struct HostHooks; + +#[async_trait] +impl IngressHooks for HostHooks { + async fn prepare( + &self, + request: &mut HttpRequest, + metadata: &mut Metadata, + _wire_format: WireFormat, + ) -> Result<(), Response> { + // Authenticate from request.headers()/extensions(), then attach trusted metadata. + let _ = (request, metadata); + Ok(()) + } + + fn resolve_model( + &self, + _request: &mut Request, + _wire_format: WireFormat, + ) -> Result { + Ok(ModelId::from("internal/route")) + } +} + +fn public_router(state: ServerState) -> axum::Router { + build_llm_router(state.with_ingress_hooks(Arc::new(HostHooks))) +} +``` + +The reduced router intentionally omits health, metrics, stats, decision, token-counting, and +routing-log endpoints. A host can mount its own protected operational surface separately. The +full [`build_switchyard_router`] remains unchanged for standalone deployments. + The server logs exactly one structured terminal event per LLM request: successful responses at `INFO`, 4xx responses at `WARN`, and 5xx responses at `ERROR`. Set `RUST_LOG=switchyard_server=debug,libsy=debug` to include routing decisions and nested failure diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 77150399c..4358ac944 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -23,8 +23,9 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::{Duration, Instant}; +use async_trait::async_trait; use axum::extract::rejection::{JsonRejection, QueryRejection}; -use axum::extract::{DefaultBodyLimit, Query, Request as HttpRequest, State}; +use axum::extract::{DefaultBodyLimit, FromRequest, Query, Request as HttpRequest, State}; use axum::http::header::CONTENT_TYPE; use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; use axum::middleware::Next; @@ -132,6 +133,52 @@ struct DecisionLlmClientResponse { base_url: String, } +/// Host hooks around the public LLM HTTP endpoints. +/// +/// The default implementation preserves the standalone server's behavior. A host may +/// authenticate before body extraction, attach trusted metadata, map an external model +/// to a route model, and present responses without copying Switchyard's handlers. +#[async_trait] +pub trait IngressHooks: Send + Sync { + /// Inspects or rejects a request before its body is read and decoded. + /// + /// `metadata` has already been normalized from request headers. Implementations may + /// add trusted host context; returning a response skips body extraction and routing. + async fn prepare( + &self, + _request: &mut HttpRequest, + _metadata: &mut Metadata, + _wire_format: WireFormat, + ) -> std::result::Result<(), Response> { + Ok(()) + } + + /// Resolves the route model after wire-format decoding and before route lookup. + /// + /// The returned model replaces `request.llm_request.model` for route execution. + #[allow(clippy::result_large_err)] + fn resolve_model( + &self, + request: &mut Request, + _wire_format: WireFormat, + ) -> std::result::Result { + request_model(request) + } + + /// Presents the final HTTP response after routing or rejection. + /// + /// The default converts Switchyard's error envelope to the caller's wire format + /// and leaves successful responses unchanged. + fn present_response(&self, response: Response, wire_format: WireFormat) -> Response { + render_error_response(response, wire_format) + } +} + +struct DefaultIngressHooks; + +#[async_trait] +impl IngressHooks for DefaultIngressHooks {} + /// Shared server state used by all endpoint handlers. #[derive(Clone)] pub struct ServerState { @@ -140,6 +187,7 @@ pub struct ServerState { stats: StatsAccumulator, routing_log: Option, track_cache_eligibility: bool, + ingress_hooks: Arc, } #[derive(Clone)] @@ -212,9 +260,16 @@ impl ServerState { stats, routing_log: None, track_cache_eligibility: tracking_enabled_from_env(), + ingress_hooks: Arc::new(DefaultIngressHooks), }) } + /// Installs host hooks for authentication, route-model resolution, and presentation. + pub fn with_ingress_hooks(mut self, hooks: Arc) -> Self { + self.ingress_hooks = hooks; + self + } + /// Enables durable per-request routing records at `path`. pub fn with_routing_log(mut self, path: impl Into) -> ServerResult { self.routing_log = Some(SharedRoutingLog::new(path.into())?); @@ -473,10 +528,7 @@ async fn stamp_request_start(mut request: HttpRequest, next: Next) -> Response { /// Builds an Axum router for the supported LLM wire formats. pub fn build_switchyard_router(state: ServerState) -> Router { - let mut router = Router::new() - .route("/v1/chat/completions", post(openai_chat_completions)) - .route("/v1/messages", post(anthropic_messages)) - .route("/v1/responses", post(openai_responses)) + let mut router = llm_routes() .route("/v1/decision", post(decision)) .route("/v1/messages/count_tokens", post(anthropic_count_tokens)) .route("/v1/models", get(models)) @@ -487,6 +539,25 @@ pub fn build_switchyard_router(state: ServerState) -> Router { if state.routing_log.is_some() { router = router.route("/v1/routing/session-stats", get(get_session_stats)); } + finish_router(router, state) +} + +/// Builds only the public Chat Completions, Responses, and Messages routes. +/// +/// Hosts can layer their own internal endpoints and middleware without exposing the +/// standalone server's metrics, stats, decision, token-counting, or health routes. +pub fn build_llm_router(state: ServerState) -> Router { + finish_router(llm_routes(), state) +} + +fn llm_routes() -> Router { + Router::new() + .route("/v1/chat/completions", post(openai_chat_completions)) + .route("/v1/messages", post(anthropic_messages)) + .route("/v1/responses", post(openai_responses)) +} + +fn finish_router(router: Router, state: ServerState) -> Router { router .fallback(not_found) .layer(DefaultBodyLimit::max(DEFAULT_MAX_REQUEST_BODY_BYTES)) @@ -515,28 +586,25 @@ fn server_io_error(error: std::io::Error) -> ServerError { async fn openai_chat_completions( State(state): State, Extension(started): Extension, - headers: HeaderMap, - body: std::result::Result, JsonRejection>, + request: HttpRequest, ) -> Response { - handle_endpoint(state, started, headers, body, WireFormat::OpenAiChat).await + handle_endpoint(state, started, request, WireFormat::OpenAiChat).await } async fn anthropic_messages( State(state): State, Extension(started): Extension, - headers: HeaderMap, - body: std::result::Result, JsonRejection>, + request: HttpRequest, ) -> Response { - handle_endpoint(state, started, headers, body, WireFormat::AnthropicMessages).await + handle_endpoint(state, started, request, WireFormat::AnthropicMessages).await } async fn openai_responses( State(state): State, Extension(started): Extension, - headers: HeaderMap, - body: std::result::Result, JsonRejection>, + request: HttpRequest, ) -> Response { - handle_endpoint(state, started, headers, body, WireFormat::OpenAiResponses).await + handle_endpoint(state, started, request, WireFormat::OpenAiResponses).await } /// One provider request submitted for a routing decision without an answer-model call. @@ -655,12 +723,11 @@ fn count_tokens_error(error: LlmClientError) -> Response { async fn handle_endpoint( state: ServerState, started: RequestStart, - headers: HeaderMap, - body: std::result::Result, JsonRejection>, + request: HttpRequest, wire_format: WireFormat, ) -> Response { - let span = observability::request_span(&headers); - handle_endpoint_inner(state, started, headers, body, wire_format) + let span = observability::request_span(request.headers()); + handle_endpoint_inner(state, started, request, wire_format) .instrument(span) .await } @@ -668,11 +735,29 @@ async fn handle_endpoint( async fn handle_endpoint_inner( state: ServerState, started: RequestStart, - headers: HeaderMap, - body: std::result::Result, JsonRejection>, + mut request: HttpRequest, wire_format: WireFormat, ) -> Response { - let metadata = metadata_from_headers(headers); + let ingress_hooks = Arc::clone(&state.ingress_hooks); + let mut metadata = metadata_from_headers(request.headers().clone()); + if let Err(response) = ingress_hooks + .prepare(&mut request, &mut metadata, wire_format) + .await + { + let mut request_log = RequestLogGuard(Some(RequestLogContext { + started: started.0, + wire_format, + requested_model: None, + streaming: false, + session_id: metadata.session_id.clone(), + correlation_id: metadata.correlation_id.clone(), + })); + let response = ingress_hooks.present_response(response, wire_format); + metrics::record_client_response(response.status().as_u16()); + request_log.emit(&response); + return response; + } + let body = Json::::from_request(request, &state).await; let routing_log_context = state .routing_log .as_ref() @@ -716,7 +801,7 @@ async fn handle_endpoint_inner( } Err((status, message)) => invalid_body_error(status, message), }; - let response = render_error_response(response, wire_format); + let response = ingress_hooks.present_response(response, wire_format); metrics::record_client_response(response.status().as_u16()); request_log.emit(&response); response @@ -751,24 +836,49 @@ fn resolve_route( body: Value, wire_format: WireFormat, ) -> std::result::Result<(&Route, Request), Response> { + resolve_route_inner(state, metadata, body, wire_format, None) +} + +#[allow(clippy::type_complexity, clippy::result_large_err)] +fn resolve_route_with_hooks( + state: &ServerState, + metadata: Metadata, + body: Value, + wire_format: WireFormat, +) -> std::result::Result<(&Route, Request), Response> { + resolve_route_inner( + state, + metadata, + body, + wire_format, + Some(state.ingress_hooks.as_ref()), + ) +} + +#[allow(clippy::type_complexity, clippy::result_large_err)] +fn resolve_route_inner<'state>( + state: &'state ServerState, + metadata: Metadata, + body: Value, + wire_format: WireFormat, + hooks: Option<&dyn IngressHooks>, +) -> std::result::Result<(&'state Route, Request), Response> { let llm_request = decode_request(wire_format, &body) .map_err(|error| invalid_body_error(StatusCode::BAD_REQUEST, error.to_string()))?; - let requested_model = llm_request - .model - .clone() - .filter(|model| !model.trim().is_empty()) - .ok_or_else(|| { - error_response( - StatusCode::BAD_REQUEST, - "request body must include a non-empty string `model`", - "invalid_request_error", - "invalid_request_error", - ) - })?; - let route = state.route_for_model(&requested_model).ok_or_else(|| { + let mut request = Request { + llm_request, + raw_request: Some(body), + metadata: Some(metadata), + }; + let route_model = match hooks { + Some(hooks) => hooks.resolve_model(&mut request, wire_format)?, + None => request_model(&request)?, + }; + request.llm_request.model = Some(route_model.to_string()); + let route = state.route_for_model(route_model.as_str()).ok_or_else(|| { error_response( StatusCode::NOT_FOUND, - format!("No route registered for model {requested_model}"), + format!("No route registered for model {route_model}"), "model_not_found", "model_not_found", ) @@ -784,19 +894,35 @@ fn resolve_route( StatusCode::BAD_REQUEST, format!( "route {requested_model} forwards an {provider} login; call it through {expected_endpoint}", + requested_model = route_model, ), "invalid_request_error", "invalid_request_error", )); } - let request = Request { - llm_request, - raw_request: Some(body), - metadata: Some(metadata), - }; Ok((route, request)) } +// Callers immediately return the error as the HTTP response; boxing it would +// add allocation and make the host hook API less natural. +#[allow(clippy::result_large_err)] +fn request_model(request: &Request) -> std::result::Result { + request + .llm_request + .model + .as_deref() + .filter(|model| !model.trim().is_empty()) + .map(ModelId::from) + .ok_or_else(|| { + error_response( + StatusCode::BAD_REQUEST, + "request body must include a non-empty string `model`", + "invalid_request_error", + "invalid_request_error", + ) + }) +} + async fn handle_llm_request( state: ServerState, started: RequestStart, @@ -806,7 +932,7 @@ async fn handle_llm_request( routing_log_context: Option, ) -> Response { let cache_probe = state.track_cache_eligibility.then(|| prefix_probe(&body)); - let (route, request) = match resolve_route(&state, metadata, body, wire_format) { + let (route, request) = match resolve_route_with_hooks(&state, metadata, body, wire_format) { Ok(resolved) => resolved, Err(response) => return response, }; diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 32290ea36..a4e4f81e4 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -8,7 +8,9 @@ use std::convert::Infallible; use std::error::Error; use std::io::Write; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use async_trait::async_trait; use axum::body::{Body, Bytes}; use axum::extract::{DefaultBodyLimit, State}; use axum::http::{HeaderMap, Request as HttpRequest, StatusCode}; @@ -22,10 +24,12 @@ use serde_json::{Value, json}; use switchyard_llm_client::{ Backend, ClientRouter, HttpBackendConfig, ModelConfig, TranslatingLlmClient, }; -use switchyard_protocol::ModelId; -use switchyard_protocol::RoutedLlmClient; +use switchyard_protocol::{Metadata, ModelId, Request as AlgorithmRequest, RoutedLlmClient}; use switchyard_server::config::load_server_state; -use switchyard_server::{DEFAULT_MAX_REQUEST_BODY_BYTES, ServerState, build_switchyard_router}; +use switchyard_server::{ + DEFAULT_MAX_REQUEST_BODY_BYTES, IngressHooks, ServerState, build_llm_router, + build_switchyard_router, +}; use tokio::net::TcpListener; use tokio::sync::Mutex; use tokio::task::JoinHandle; @@ -477,6 +481,91 @@ async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Route Ok((upstream, app)) } +struct TestIngressHooks { + prepared: Arc, + reject: bool, +} + +#[async_trait] +impl IngressHooks for TestIngressHooks { + async fn prepare( + &self, + _request: &mut axum::extract::Request, + _metadata: &mut Metadata, + _wire_format: switchyard_protocol::WireFormat, + ) -> Result<(), HttpResponse> { + self.prepared.fetch_add(1, Ordering::SeqCst); + if self.reject { + return Err(StatusCode::UNAUTHORIZED.into_response()); + } + Ok(()) + } + + fn resolve_model( + &self, + request: &mut AlgorithmRequest, + _wire_format: switchyard_protocol::WireFormat, + ) -> Result { + request.llm_request.model = Some(ROUTE_MODEL.to_string()); + Ok(ModelId::from(ROUTE_MODEL)) + } +} + +#[tokio::test] +async fn ingress_hook_can_reject_before_json_body_extraction() -> TestResult { + let upstream = MockUpstream::start().await?; + let prepared = Arc::new(AtomicUsize::new(0)); + let state = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/a"])])? + .with_ingress_hooks(Arc::new(TestIngressHooks { + prepared: Arc::clone(&prepared), + reject: true, + })); + let app = build_llm_router(state); + + let response = send_raw_json( + &app, + "/v1/chat/completions", + b"not-json".to_vec(), + Some("application/json"), + ) + .await?; + + assert_eq!(response.status, StatusCode::UNAUTHORIZED); + assert_eq!(prepared.load(Ordering::SeqCst), 1); + assert!(upstream.models().await.is_empty()); + Ok(()) +} + +#[tokio::test] +async fn ingress_hook_maps_public_model_and_llm_router_excludes_internal_endpoints() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/a"])])? + .with_ingress_hooks(Arc::new(TestIngressHooks { + prepared: Arc::new(AtomicUsize::new(0)), + reject: false, + })); + let app = build_llm_router(state); + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "public-model", + "messages": [{"role": "user", "content": "hello"}] + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + assert_eq!(upstream.models().await, vec!["model/a"]); + assert_eq!( + send(&app, "GET", "/metrics", None).await?.status, + StatusCode::NOT_FOUND + ); + Ok(()) +} + fn empty_token_totals() -> Value { json!({ "prompt": 0, From 4f6a8b80f0bc5f45266d2da5521ef1ff3c4cea0b Mon Sep 17 00:00:00 2001 From: Ilexpwh <122920329+Ilexpwh@users.noreply.github.com> Date: Sun, 30 Aug 2026 14:39:40 +0800 Subject: [PATCH 2/6] docs(server): clarify ingress hook contracts Signed-off-by: Ilexpwh <122920329+Ilexpwh@users.noreply.github.com> --- crates/switchyard-server/README.md | 8 ++++++++ crates/switchyard-server/src/lib.rs | 2 ++ crates/switchyard-server/tests/server.rs | 2 ++ 3 files changed, 12 insertions(+) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index fdf0e13c8..b8b72e5ab 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -70,6 +70,14 @@ host can authenticate or reject without buffering an untrusted body. `resolve_mo wire-format decoding and can map a public model to an internal route. `present_response` can adapt the final error envelope. All methods have defaults that preserve the standalone server behavior. +The embedding crate must declare `async-trait` as a direct dependency before implementing the +hook trait: + +```toml +[dependencies] +async-trait = "0.1" +``` + ```rust use std::sync::Arc; diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 4358ac944..2d4a04ade 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -156,6 +156,8 @@ pub trait IngressHooks: Send + Sync { /// Resolves the route model after wire-format decoding and before route lookup. /// /// The returned model replaces `request.llm_request.model` for route execution. + /// Returning `Err(response)` stops route lookup and uses that response as the + /// endpoint response, allowing a host to reject an already decoded request. #[allow(clippy::result_large_err)] fn resolve_model( &self, diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index a4e4f81e4..97908e541 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -511,6 +511,7 @@ impl IngressHooks for TestIngressHooks { } } +// A prepare rejection must occur before malformed JSON is read and decoded. #[tokio::test] async fn ingress_hook_can_reject_before_json_body_extraction() -> TestResult { let upstream = MockUpstream::start().await?; @@ -536,6 +537,7 @@ async fn ingress_hook_can_reject_before_json_body_extraction() -> TestResult { Ok(()) } +// Public models map to internal routes while the restricted router omits operational endpoints. #[tokio::test] async fn ingress_hook_maps_public_model_and_llm_router_excludes_internal_endpoints() -> TestResult { let upstream = MockUpstream::start().await?; From 2dcb81317342098f68123226a48ac683d14d4589 Mon Sep 17 00:00:00 2001 From: Ilexpwh <122920329+Ilexpwh@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:04:56 +0800 Subject: [PATCH 3/6] feat(server): await decoded-request ingress policy Signed-off-by: Ilexpwh <122920329+Ilexpwh@users.noreply.github.com> --- crates/switchyard-server/README.md | 9 +++-- crates/switchyard-server/src/lib.rs | 51 +++++++++++++----------- crates/switchyard-server/tests/server.rs | 46 ++++++++++++++++++++- 3 files changed, 78 insertions(+), 28 deletions(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index b8b72e5ab..37000d3e2 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -66,9 +66,10 @@ Ctrl+C and Unix `SIGTERM` stop new connections and allow active requests to drai Rust hosts can install [`IngressHooks`] on [`ServerState`] and use [`build_llm_router`] to expose only Chat Completions, Responses, and Messages. `prepare` runs before JSON body extraction, so a -host can authenticate or reject without buffering an untrusted body. `resolve_model` runs after -wire-format decoding and can map a public model to an internal route. `present_response` can adapt -the final error envelope. All methods have defaults that preserve the standalone server behavior. +host can authenticate or reject without buffering an untrusted body. The async `resolve_model` +runs after wire-format decoding and can enforce host policy before mapping a public model to an +internal route. `present_response` can adapt the final error envelope. All methods have defaults +that preserve the standalone server behavior. The embedding crate must declare `async-trait` as a direct dependency before implementing the hook trait: @@ -102,7 +103,7 @@ impl IngressHooks for HostHooks { Ok(()) } - fn resolve_model( + async fn resolve_model( &self, _request: &mut Request, _wire_format: WireFormat, diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 2d4a04ade..9f3a9a918 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -153,13 +153,13 @@ pub trait IngressHooks: Send + Sync { Ok(()) } - /// Resolves the route model after wire-format decoding and before route lookup. + /// Resolves the route model asynchronously after wire-format decoding and before route lookup. /// /// The returned model replaces `request.llm_request.model` for route execution. /// Returning `Err(response)` stops route lookup and uses that response as the /// endpoint response, allowing a host to reject an already decoded request. #[allow(clippy::result_large_err)] - fn resolve_model( + async fn resolve_model( &self, request: &mut Request, _wire_format: WireFormat, @@ -838,44 +838,48 @@ fn resolve_route( body: Value, wire_format: WireFormat, ) -> std::result::Result<(&Route, Request), Response> { - resolve_route_inner(state, metadata, body, wire_format, None) + let request = decode_route_request(metadata, body, wire_format)?; + let route_model = request_model(&request)?; + resolve_decoded_route(state, request, route_model, wire_format) } #[allow(clippy::type_complexity, clippy::result_large_err)] -fn resolve_route_with_hooks( +async fn resolve_route_with_hooks( state: &ServerState, metadata: Metadata, body: Value, wire_format: WireFormat, ) -> std::result::Result<(&Route, Request), Response> { - resolve_route_inner( - state, - metadata, - body, - wire_format, - Some(state.ingress_hooks.as_ref()), - ) + let mut request = decode_route_request(metadata, body, wire_format)?; + let route_model = state + .ingress_hooks + .resolve_model(&mut request, wire_format) + .await?; + resolve_decoded_route(state, request, route_model, wire_format) } -#[allow(clippy::type_complexity, clippy::result_large_err)] -fn resolve_route_inner<'state>( - state: &'state ServerState, +#[allow(clippy::result_large_err)] +fn decode_route_request( metadata: Metadata, body: Value, wire_format: WireFormat, - hooks: Option<&dyn IngressHooks>, -) -> std::result::Result<(&'state Route, Request), Response> { +) -> std::result::Result { let llm_request = decode_request(wire_format, &body) .map_err(|error| invalid_body_error(StatusCode::BAD_REQUEST, error.to_string()))?; - let mut request = Request { + Ok(Request { llm_request, raw_request: Some(body), metadata: Some(metadata), - }; - let route_model = match hooks { - Some(hooks) => hooks.resolve_model(&mut request, wire_format)?, - None => request_model(&request)?, - }; + }) +} + +#[allow(clippy::type_complexity, clippy::result_large_err)] +fn resolve_decoded_route( + state: &ServerState, + mut request: Request, + route_model: ModelId, + wire_format: WireFormat, +) -> std::result::Result<(&Route, Request), Response> { request.llm_request.model = Some(route_model.to_string()); let route = state.route_for_model(route_model.as_str()).ok_or_else(|| { error_response( @@ -934,7 +938,8 @@ async fn handle_llm_request( routing_log_context: Option, ) -> Response { let cache_probe = state.track_cache_eligibility.then(|| prefix_probe(&body)); - let (route, request) = match resolve_route_with_hooks(&state, metadata, body, wire_format) { + let (route, request) = match resolve_route_with_hooks(&state, metadata, body, wire_format).await + { Ok(resolved) => resolved, Err(response) => return response, }; diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 97908e541..53201d78b 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -483,7 +483,9 @@ async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Route struct TestIngressHooks { prepared: Arc, + resolved: Arc, reject: bool, + reject_resolve: bool, } #[async_trait] @@ -501,11 +503,16 @@ impl IngressHooks for TestIngressHooks { Ok(()) } - fn resolve_model( + async fn resolve_model( &self, request: &mut AlgorithmRequest, _wire_format: switchyard_protocol::WireFormat, ) -> Result { + tokio::task::yield_now().await; + self.resolved.fetch_add(1, Ordering::SeqCst); + if self.reject_resolve { + return Err(StatusCode::FORBIDDEN.into_response()); + } request.llm_request.model = Some(ROUTE_MODEL.to_string()); Ok(ModelId::from(ROUTE_MODEL)) } @@ -519,7 +526,9 @@ async fn ingress_hook_can_reject_before_json_body_extraction() -> TestResult { let state = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/a"])])? .with_ingress_hooks(Arc::new(TestIngressHooks { prepared: Arc::clone(&prepared), + resolved: Arc::new(AtomicUsize::new(0)), reject: true, + reject_resolve: false, })); let app = build_llm_router(state); @@ -541,10 +550,13 @@ async fn ingress_hook_can_reject_before_json_body_extraction() -> TestResult { #[tokio::test] async fn ingress_hook_maps_public_model_and_llm_router_excludes_internal_endpoints() -> TestResult { let upstream = MockUpstream::start().await?; + let resolved = Arc::new(AtomicUsize::new(0)); let state = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/a"])])? .with_ingress_hooks(Arc::new(TestIngressHooks { prepared: Arc::new(AtomicUsize::new(0)), + resolved: Arc::clone(&resolved), reject: false, + reject_resolve: false, })); let app = build_llm_router(state); @@ -560,6 +572,7 @@ async fn ingress_hook_maps_public_model_and_llm_router_excludes_internal_endpoin .await?; assert_eq!(response.status, StatusCode::OK); + assert_eq!(resolved.load(Ordering::SeqCst), 1); assert_eq!(upstream.models().await, vec!["model/a"]); assert_eq!( send(&app, "GET", "/metrics", None).await?.status, @@ -568,6 +581,37 @@ async fn ingress_hook_maps_public_model_and_llm_router_excludes_internal_endpoin Ok(()) } +// An asynchronous decoded-request rejection must finish before route execution. +#[tokio::test] +async fn ingress_hook_can_reject_after_decode_before_route_execution() -> TestResult { + let upstream = MockUpstream::start().await?; + let resolved = Arc::new(AtomicUsize::new(0)); + let state = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/a"])])? + .with_ingress_hooks(Arc::new(TestIngressHooks { + prepared: Arc::new(AtomicUsize::new(0)), + resolved: Arc::clone(&resolved), + reject: false, + reject_resolve: true, + })); + let app = build_llm_router(state); + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "public-model", + "messages": [{"role": "user", "content": "hello"}] + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::FORBIDDEN); + assert_eq!(resolved.load(Ordering::SeqCst), 1); + assert!(upstream.models().await.is_empty()); + Ok(()) +} + fn empty_token_totals() -> Value { json!({ "prompt": 0, From c892b891663fdb78a474ab5a7fa7cf293f76a886 Mon Sep 17 00:00:00 2001 From: Ilexpwh <122920329+Ilexpwh@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:15:09 +0800 Subject: [PATCH 4/6] feat(server): expose typed API error metadata Signed-off-by: Ilexpwh <122920329+Ilexpwh@users.noreply.github.com> --- crates/switchyard-server/README.md | 4 ++ crates/switchyard-server/src/lib.rs | 30 +++++++++++++-- crates/switchyard-server/tests/server.rs | 48 +++++++++++++++++++++++- 3 files changed, 76 insertions(+), 6 deletions(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 37000d3e2..1b46ae6c6 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -71,6 +71,10 @@ runs after wire-format decoding and can enforce host policy before mapping a pub internal route. `present_response` can adapt the final error envelope. All methods have defaults that preserve the standalone server behavior. +Server-generated failures retain public [`ApiError`] metadata in response extensions. A host +presenter can inspect its status, message, type, and stable code without buffering or parsing the +response body. + The embedding crate must declare `async-trait` as a direct dependency before implementing the hook trait: diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 9f3a9a918..8a32d1a79 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -1204,9 +1204,9 @@ fn upstream_error_message(body: &str) -> String { .unwrap_or_else(|| body.to_string()) } -// Error metadata retained until the client-facing endpoint selects an envelope. +/// Structured server error retained until an ingress presenter selects an envelope. #[derive(Clone)] -struct ApiError { +pub struct ApiError { status: StatusCode, message: String, error_type: &'static str, @@ -1214,7 +1214,8 @@ struct ApiError { } impl ApiError { - fn new( + /// Creates error metadata that can be rendered for any supported wire format. + pub fn new( status: StatusCode, message: impl Into, error_type: &'static str, @@ -1228,7 +1229,28 @@ impl ApiError { } } - fn into_response(self, wire_format: WireFormat) -> Response { + /// Returns the HTTP status associated with this error. + pub fn status(&self) -> StatusCode { + self.status + } + + /// Returns the client-facing error message. + pub fn message(&self) -> &str { + &self.message + } + + /// Returns the OpenAI-compatible error type. + pub fn error_type(&self) -> &'static str { + self.error_type + } + + /// Returns the stable machine-readable error code. + pub fn code(&self) -> &'static str { + self.code + } + + /// Renders this error and preserves its metadata for a later presenter. + pub fn into_response(self, wire_format: WireFormat) -> Response { let body = match wire_format { WireFormat::AnthropicMessages => json!({ "type": "error", diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 53201d78b..880b83016 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -13,7 +13,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use async_trait::async_trait; use axum::body::{Body, Bytes}; use axum::extract::{DefaultBodyLimit, State}; -use axum::http::{HeaderMap, Request as HttpRequest, StatusCode}; +use axum::http::{HeaderMap, HeaderValue, Request as HttpRequest, StatusCode}; use axum::response::sse::{Event, Sse}; use axum::response::{IntoResponse, Response as HttpResponse}; use axum::routing::post; @@ -27,7 +27,7 @@ use switchyard_llm_client::{ use switchyard_protocol::{Metadata, ModelId, Request as AlgorithmRequest, RoutedLlmClient}; use switchyard_server::config::load_server_state; use switchyard_server::{ - DEFAULT_MAX_REQUEST_BODY_BYTES, IngressHooks, ServerState, build_llm_router, + ApiError, DEFAULT_MAX_REQUEST_BODY_BYTES, IngressHooks, ServerState, build_llm_router, build_switchyard_router, }; use tokio::net::TcpListener; @@ -516,6 +516,20 @@ impl IngressHooks for TestIngressHooks { request.llm_request.model = Some(ROUTE_MODEL.to_string()); Ok(ModelId::from(ROUTE_MODEL)) } + + fn present_response( + &self, + mut response: HttpResponse, + _wire_format: switchyard_protocol::WireFormat, + ) -> HttpResponse { + let error_code = response.extensions().get::().map(ApiError::code); + if let Some(error_code) = error_code { + response + .headers_mut() + .insert("x-test-error-code", HeaderValue::from_static(error_code)); + } + response + } } // A prepare rejection must occur before malformed JSON is read and decoded. @@ -612,6 +626,36 @@ async fn ingress_hook_can_reject_after_decode_before_route_execution() -> TestRe Ok(()) } +// A host presenter can inspect structured errors without reading or replacing the body. +#[tokio::test] +async fn ingress_hook_can_present_structured_server_errors() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/a"])])? + .with_ingress_hooks(Arc::new(TestIngressHooks { + prepared: Arc::new(AtomicUsize::new(0)), + resolved: Arc::new(AtomicUsize::new(0)), + reject: false, + reject_resolve: false, + })); + let app = build_llm_router(state); + + let response = send_raw_json( + &app, + "/v1/chat/completions", + b"not-json".to_vec(), + Some("application/json"), + ) + .await?; + + assert_eq!(response.status, StatusCode::BAD_REQUEST); + assert_eq!( + response.headers.get("x-test-error-code"), + Some(&HeaderValue::from_static("invalid_body")) + ); + assert!(upstream.models().await.is_empty()); + Ok(()) +} + fn empty_token_totals() -> Value { json!({ "prompt": 0, From 3f66c9ae304a9094272d227620a727bd12d80e1c Mon Sep 17 00:00:00 2001 From: Ilexpwh <122920329+Ilexpwh@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:23:43 +0800 Subject: [PATCH 5/6] feat(server): present resolved request context Signed-off-by: Ilexpwh <122920329+Ilexpwh@users.noreply.github.com> --- crates/switchyard-server/README.md | 14 ++++++++-- crates/switchyard-server/src/lib.rs | 33 ++++++++++++++++-------- crates/switchyard-server/tests/server.rs | 23 +++++++++++++++++ 3 files changed, 57 insertions(+), 13 deletions(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 1b46ae6c6..a7b483803 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -68,8 +68,9 @@ Rust hosts can install [`IngressHooks`] on [`ServerState`] and use [`build_llm_r only Chat Completions, Responses, and Messages. `prepare` runs before JSON body extraction, so a host can authenticate or reject without buffering an untrusted body. The async `resolve_model` runs after wire-format decoding and can enforce host policy before mapping a public model to an -internal route. `present_response` can adapt the final error envelope. All methods have defaults -that preserve the standalone server behavior. +internal route. `present_response` receives the resolved request metadata and can adapt the final +error envelope or add request-scoped headers. All methods have defaults that preserve the +standalone server behavior. Server-generated failures retain public [`ApiError`] metadata in response extensions. A host presenter can inspect its status, message, type, and stable code without buffering or parsing the @@ -114,6 +115,15 @@ impl IngressHooks for HostHooks { ) -> Result { Ok(ModelId::from("internal/route")) } + + fn present_response( + &self, + response: Response, + _wire_format: WireFormat, + _request_metadata: &Metadata, + ) -> Response { + response + } } fn public_router(state: ServerState) -> axum::Router { diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 8a32d1a79..9bf5829c8 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -169,9 +169,16 @@ pub trait IngressHooks: Send + Sync { /// Presents the final HTTP response after routing or rejection. /// + /// `request_metadata` includes trusted changes made by `prepare` and + /// `resolve_model`, allowing request-scoped headers without global state. /// The default converts Switchyard's error envelope to the caller's wire format /// and leaves successful responses unchanged. - fn present_response(&self, response: Response, wire_format: WireFormat) -> Response { + fn present_response( + &self, + response: Response, + wire_format: WireFormat, + _request_metadata: &Metadata, + ) -> Response { render_error_response(response, wire_format) } } @@ -754,7 +761,7 @@ async fn handle_endpoint_inner( session_id: metadata.session_id.clone(), correlation_id: metadata.correlation_id.clone(), })); - let response = ingress_hooks.present_response(response, wire_format); + let response = ingress_hooks.present_response(response, wire_format, &metadata); metrics::record_client_response(response.status().as_u16()); request_log.emit(&response); return response; @@ -794,7 +801,7 @@ async fn handle_endpoint_inner( handle_llm_request( state, started, - metadata, + &mut metadata, body, wire_format, routing_log_context, @@ -803,7 +810,7 @@ async fn handle_endpoint_inner( } Err((status, message)) => invalid_body_error(status, message), }; - let response = ingress_hooks.present_response(response, wire_format); + let response = ingress_hooks.present_response(response, wire_format, &metadata); metrics::record_client_response(response.status().as_u16()); request_log.emit(&response); response @@ -844,17 +851,21 @@ fn resolve_route( } #[allow(clippy::type_complexity, clippy::result_large_err)] -async fn resolve_route_with_hooks( - state: &ServerState, - metadata: Metadata, +async fn resolve_route_with_hooks<'state>( + state: &'state ServerState, + metadata: &mut Metadata, body: Value, wire_format: WireFormat, -) -> std::result::Result<(&Route, Request), Response> { - let mut request = decode_route_request(metadata, body, wire_format)?; +) -> std::result::Result<(&'state Route, Request), Response> { + let mut request = decode_route_request(metadata.clone(), body, wire_format)?; let route_model = state .ingress_hooks .resolve_model(&mut request, wire_format) - .await?; + .await; + if let Some(resolved_metadata) = request.metadata.as_ref() { + *metadata = resolved_metadata.clone(); + } + let route_model = route_model?; resolve_decoded_route(state, request, route_model, wire_format) } @@ -932,7 +943,7 @@ fn request_model(request: &Request) -> std::result::Result { async fn handle_llm_request( state: ServerState, started: RequestStart, - metadata: Metadata, + metadata: &mut Metadata, body: Value, wire_format: WireFormat, routing_log_context: Option, diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 880b83016..4c81054f0 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -488,6 +488,9 @@ struct TestIngressHooks { reject_resolve: bool, } +#[derive(Clone)] +struct ResolvedPresentationMarker; + #[async_trait] impl IngressHooks for TestIngressHooks { async fn prepare( @@ -510,6 +513,11 @@ impl IngressHooks for TestIngressHooks { ) -> Result { tokio::task::yield_now().await; self.resolved.fetch_add(1, Ordering::SeqCst); + request + .metadata + .get_or_insert_default() + .typed_context + .insert(ResolvedPresentationMarker); if self.reject_resolve { return Err(StatusCode::FORBIDDEN.into_response()); } @@ -521,6 +529,7 @@ impl IngressHooks for TestIngressHooks { &self, mut response: HttpResponse, _wire_format: switchyard_protocol::WireFormat, + request_metadata: &Metadata, ) -> HttpResponse { let error_code = response.extensions().get::().map(ApiError::code); if let Some(error_code) = error_code { @@ -528,6 +537,16 @@ impl IngressHooks for TestIngressHooks { .headers_mut() .insert("x-test-error-code", HeaderValue::from_static(error_code)); } + if request_metadata + .typed_context + .get::() + .is_some() + { + response.headers_mut().insert( + "x-test-resolved-context", + HeaderValue::from_static("present"), + ); + } response } } @@ -586,6 +605,10 @@ async fn ingress_hook_maps_public_model_and_llm_router_excludes_internal_endpoin .await?; assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response.headers.get("x-test-resolved-context"), + Some(&HeaderValue::from_static("present")) + ); assert_eq!(resolved.load(Ordering::SeqCst), 1); assert_eq!(upstream.models().await, vec!["model/a"]); assert_eq!( From 349c8851b1fadb3923b59e6dbfbca7513f2cdd7e Mon Sep 17 00:00:00 2001 From: ilexpoon Date: Mon, 31 Aug 2026 15:43:24 +0800 Subject: [PATCH 6/6] feat(server): present host-selected response model Signed-off-by: Ilexpwh <122920329+Ilexpwh@users.noreply.github.com> --- crates/switchyard-server/README.md | 13 +++++++++++++ crates/switchyard-server/src/lib.rs | 18 +++++++++++++++++- crates/switchyard-server/tests/server.rs | 13 +++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index a7b483803..4bb094ae4 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -72,6 +72,11 @@ internal route. `present_response` receives the resolved request metadata and ca error envelope or add request-scoped headers. All methods have defaults that preserve the standalone server behavior. +`presented_response_model` runs before response encoding. It receives the selected provider +model and resolved request metadata, allowing a host to restore a public model name without +buffering or parsing JSON/SSE response bodies. Metrics and routing records remain keyed by the +actual served model. + Server-generated failures retain public [`ApiError`] metadata in response extensions. A host presenter can inspect its status, message, type, and stable code without buffering or parsing the response body. @@ -116,6 +121,14 @@ impl IngressHooks for HostHooks { Ok(ModelId::from("internal/route")) } + fn presented_response_model( + &self, + _served_model: Option<&ModelId>, + _request_metadata: &Metadata, + ) -> Option { + Some("public-model".to_string()) + } + fn present_response( &self, response: Response, diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 9bf5829c8..8cc4c850a 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -167,6 +167,20 @@ pub trait IngressHooks: Send + Sync { request_model(request) } + /// Selects the model identifier encoded in a successful response. + /// + /// `served_model` is the selected provider model used by the standalone server. + /// Hosts that map public models to private routes can instead return the public + /// model retained in trusted `request_metadata`. This affects response encoding + /// only; metrics and routing records continue to use the actual served model. + fn presented_response_model( + &self, + served_model: Option<&ModelId>, + _request_metadata: &Metadata, + ) -> Option { + served_model.map(ToString::to_string) + } + /// Presents the final HTTP response after routing or rejection. /// /// `request_metadata` includes trusted changes made by `prepare` and @@ -971,6 +985,9 @@ async fn handle_llm_request( // The response carries the candidate that actually served it. Fall back to the routing // selection for algorithms that return a response without an offloaded model call. let served_model = response.served_model().cloned().or(Some(selected_model)); + let response_model = state + .ingress_hooks + .presented_response_model(served_model.as_ref(), metadata); let response = if let Some(served_model) = served_model.as_ref() { let cache_eligible = cache_probe .as_ref() @@ -988,7 +1005,6 @@ async fn handle_llm_request( response }; - let response_model = served_model.as_ref().map(ToString::to_string); let mut response = match into_http_response(response, wire_format, response_model, request_extensions) { Ok(response) => response, diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 4c81054f0..febfa5499 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -525,6 +525,18 @@ impl IngressHooks for TestIngressHooks { Ok(ModelId::from(ROUTE_MODEL)) } + fn presented_response_model( + &self, + served_model: Option<&ModelId>, + request_metadata: &Metadata, + ) -> Option { + assert_eq!(served_model.map(ModelId::as_str), Some("model/a")); + request_metadata + .typed_context + .get::() + .map(|_| "public-model".to_string()) + } + fn present_response( &self, mut response: HttpResponse, @@ -605,6 +617,7 @@ async fn ingress_hook_maps_public_model_and_llm_router_excludes_internal_endpoin .await?; assert_eq!(response.status, StatusCode::OK); + assert_eq!(response.json()?["model"], "public-model"); assert_eq!( response.headers.get("x-test-resolved-context"), Some(&HeaderValue::from_static("present"))