diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index a601ab94a..2940f8a7c 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -13,8 +13,7 @@ use libsy::{ ClassifyTrigger, CompositeRouter, CompositeRouterConfig, CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, - StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TargetPrompts, - TaskClassifierConfig, + StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, }; use serde::Deserialize; use switchyard_protocol::ModelId; @@ -378,12 +377,6 @@ pub struct StageTierConfig { /// Notes handed to a tier when the router switches to it. #[serde(default)] pub handoff_notes: Option, - /// System prompt handed to the capable tier. - #[serde(default)] - pub capable_system_prompt: Option, - /// System prompt handed to the efficient tier. - #[serde(default)] - pub efficient_system_prompt: Option, } impl StageClassifierConfig { @@ -513,6 +506,39 @@ impl AlgorithmSpec { } names } + + /// Response target and routing-only dependency for routers that answer while routing. + pub(crate) fn routing_response_and_dependency(&self) -> Option<(&str, &str)> { + match self { + Self::LlmClassifier { config, .. } + if matches!( + config.mode.unwrap_or(if config.escalation.is_some() { + ClassifierMode::Escalation + } else { + ClassifierMode::Capability + }), + ClassifierMode::Escalation + ) => + { + Some(( + config.weak_target.as_deref()?, + config.classifier_target.as_str(), + )) + } + Self::Advisor { + executor_target, + advisor_target, + .. + } => Some((executor_target, advisor_target)), + Self::Noop { .. } + | Self::Random { .. } + | Self::Passthrough { .. } + | Self::LlmClassifier { .. } + | Self::StageRouter { .. } + | Self::Composite { .. } => None, + } + } + /// Builds this algorithm after resolving configured target names. pub fn build( &self, @@ -945,8 +971,6 @@ fn build_algorithm( confidence_threshold, recent_turn_window, handoff_notes, - capable_system_prompt, - efficient_system_prompt, } = tiers; if matches!(picker, PickerMode::CapableFirst) { tracing::warn!( @@ -958,12 +982,6 @@ fn build_algorithm( let mut config = StageRouterConfig::new(*picker, *confidence_threshold); config.recent_window = *recent_turn_window; config.handoff_notes = handoff_notes.clone(); - config.tier_prompts = tier_prompts( - &capable, - capable_system_prompt.as_deref(), - &efficient, - efficient_system_prompt.as_deref(), - ); // The judge is called through its own target, so it is not a routing // destination and stays out of the tier pair. config.llm_fallback = classifier @@ -998,12 +1016,6 @@ fn build_algorithm( StageRouterConfig::new(PickerMode::EfficientFirst, stage.confidence_threshold); stage_config.recent_window = stage.recent_turn_window; stage_config.handoff_notes = stage.handoff_notes.clone(); - stage_config.tier_prompts = tier_prompts( - &capable, - stage.capable_system_prompt.as_deref(), - &efficient, - stage.efficient_system_prompt.as_deref(), - ); let config = CompositeRouterConfig { judge_target, judge: classifier.task_classifier_config(), @@ -1101,23 +1113,6 @@ fn default_classifier_max_output_tokens() -> u64 { TaskClassifierConfig::default().max_output_tokens } -/// Keys each configured system prompt by the target it belongs to. -fn tier_prompts( - capable: &str, - capable_prompt: Option<&str>, - efficient: &str, - efficient_prompt: Option<&str>, -) -> TargetPrompts { - let mut prompts = TargetPrompts::default(); - if let Some(prompt) = capable_prompt { - prompts = prompts.with(capable, prompt); - } - if let Some(prompt) = efficient_prompt { - prompts = prompts.with(efficient, prompt); - } - prompts -} - fn resolve_targets<'a>( route_name: &str, names: impl IntoIterator, diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 4b211ae81..eb2c513c5 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -295,7 +295,60 @@ impl DeploymentConfig { let client: Arc = client.clone(); by_model.insert(target.id.clone(), client); } - Ok((ClientRouter::new(by_model), caller_auth)) + let (target_prompts, routing_answer_target) = + self.build_route_target_prompts(route_name, route)?; + let router = + ClientRouter::new_with_target_prompts(by_model, target_prompts, routing_answer_target); + Ok((router, caller_auth)) + } + + /// Builds the effective prompt policy for this route's completion targets. + fn build_route_target_prompts( + &self, + route_name: &str, + route: &RouteConfig, + ) -> RunnerResult<(HashMap, Option)> { + let mut prompts = HashMap::new(); + let mut aliases = HashMap::<&ModelId, Option<&str>>::new(); + for name in route.algorithm.routing_target_names() { + let target = self.targets.get(name).ok_or_else(|| { + RunnerError::configuration(format!("route references unknown target {name}")) + })?; + let prompt = target.system_prompt.as_deref(); + if aliases + .insert(&target.id, prompt) + .is_some_and(|configured| configured != prompt) + { + return Err(RunnerError::configuration(format!( + "route {route_name} maps completion target aliases to model {} with different system_prompt values", + target.id + ))); + } + if let Some(prompt) = prompt { + prompts.insert(target.id.clone(), prompt.to_string()); + } + } + let Some((response_name, dependency_name)) = + route.algorithm.routing_response_and_dependency() + else { + return Ok((prompts, None)); + }; + let response = self.targets.get(response_name).ok_or_else(|| { + RunnerError::configuration(format!("route references unknown target {response_name}")) + })?; + if !prompts.contains_key(&response.id) { + return Ok((prompts, None)); + } + let dependency = self.targets.get(dependency_name).ok_or_else(|| { + RunnerError::configuration(format!("route references unknown target {dependency_name}")) + })?; + if response.id == dependency.id { + return Err(RunnerError::configuration(format!( + "route {route_name} cannot apply system_prompt to target {response_name}: model {} is also used by routing-only target {dependency_name}", + response.id, + ))); + } + Ok((prompts, Some(response.id.clone()))) } fn fallback_base_url(&self) -> RunnerResult> { @@ -419,6 +472,7 @@ struct TargetConfig { llm_client: String, #[serde(default)] extra_body: BTreeMap, + system_prompt: Option, } #[derive(Clone, Copy, Debug, Deserialize)] @@ -764,6 +818,49 @@ confidence_threshold = 0.5 Ok(()) } + #[test] + fn aliased_completion_targets_reject_prompt_conflicts() { + let configured = stage_config() + .replace( + "id = \"strong/model\"\nllm_client = \"responses\"", + "id = \"strong/model\"\nllm_client = \"responses\"\nsystem_prompt = \"capable\"", + ) + .replace( + "[routes.stage]", + "[targets.strong_alias]\nid = \"strong/model\"\nllm_client = \"responses\"\n\n[routes.stage]", + ) + .replace("efficient_target = \"weak\"", "efficient_target = \"strong_alias\""); + let message = error_message(&configured); + assert!( + message.contains("completion target aliases to model strong/model with different system_prompt values"), + "unexpected error: {message}" + ); + } + + #[test] + fn prompted_routing_response_cannot_share_a_model_with_a_dependency() { + let configured = VALID_CONFIG + .replace( + "id = \"classifier/model\"\nllm_client = \"primary\"", + "id = \"weak/model\"\nllm_client = \"primary\"", + ) + .replace( + "id = \"weak/model\"\nllm_client = \"anthropic\"", + "id = \"weak/model\"\nllm_client = \"anthropic\"\nsystem_prompt = \"answer prompt\"", + ) + .replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 1 }", + ); + + let message = error_message(&configured); + + assert!( + message.contains("cannot apply system_prompt to target weak: model weak/model is also used by routing-only target classifier"), + "unexpected error: {message}" + ); + } + #[test] fn rejects_invalid_unreferenced_llm_client() { let invalid = format!( diff --git a/crates/switchyard-server/CONFIGURATION.md b/crates/switchyard-server/CONFIGURATION.md index ae40d38eb..f7458bc95 100644 --- a/crates/switchyard-server/CONFIGURATION.md +++ b/crates/switchyard-server/CONFIGURATION.md @@ -14,9 +14,13 @@ max_retries = 2 [targets.model] id = "provider/model" llm_client = "provider" +system_prompt = "Follow this model's deployment instructions." extra_body = { chat_template_kwargs = { enable_thinking = false } } ``` +`system_prompt` is prepended when the target is a completion destination. Switchyard +prepares each fallback independently, so a failed target's prompt is not carried to the next one. + `extra_body` is target-specific. It shallow-merges top-level provider options into the outbound request, while explicit request fields win on conflicts. diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 3f59fd0af..aa76c43c2 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -17,6 +17,7 @@ max_retries = 2 [targets.model_a] id = "model/a" llm_client = "example" +system_prompt = "Use the fast path for routine work." extra_body = { service_tier = "priority" } [targets.model_b] @@ -83,6 +84,8 @@ client's `base_url` should receive the caller's login. A forwarding route must be called through the matching provider API. Target-level `extra_body` values are shallow-merged into the upstream request when the request does not already contain that key. +Target-level `system_prompt` values are prepended when that target serves a completion. +Selected and fallback targets are prepared independently. `max_retries` defaults to `2` and applies to transport failures, timeouts, HTTP 408/429, and 5xx responses. @@ -166,7 +169,8 @@ target and summarizes its score, confidence, and input-dimension histograms. The with `/v1/stats/reset`; the process-lifetime counters on `/metrics` remain cumulative. Token counting selects an Anthropic-format completion target, preferring target names or model IDs -containing `opus`, `sonnet`, then `haiku`. Other ties preserve the route's target order. +containing `opus`, `sonnet`, then `haiku`. Other ties preserve the route's target order. Target +system prompts are applied to answer calls, not token-count requests. ## Metrics diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 4fca26f02..9b348dbc1 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -97,19 +97,36 @@ impl Drop for MockUpstream { } } +fn user_prompt(body: &Value) -> &str { + body["messages"] + .as_array() + .and_then(|messages| messages.iter().find(|message| message["role"] == "user")) + .and_then(|message| message["content"].as_str()) + .unwrap_or_default() +} + +fn has_system_prompt(call: &Value, expected: &str) -> bool { + call["messages"].as_array().is_some_and(|messages| { + messages.iter().any(|message| { + message["role"] == "system" && message["content"].as_str() == Some(expected) + }) + }) +} + async fn upstream_chat( State(calls): State>>>, Json(body): Json, ) -> HttpResponse { calls.lock().await.push(body.clone()); - if body["messages"][0]["content"] == "fail" { + let prompt = user_prompt(&body); + if prompt == "fail" { return ( StatusCode::IM_A_TEAPOT, Json(json!({"error": {"message": "upstream rejected request"}})), ) .into_response(); } - if body["messages"][0]["content"] == "auth-fail" { + if prompt == "auth-fail" { return ( StatusCode::UNAUTHORIZED, Json(json!({"error": {"message": "upstream authentication failed"}})), @@ -118,13 +135,12 @@ async fn upstream_chat( } let model = body["model"].as_str().unwrap_or("unknown").to_string(); - let prompt = body["messages"][0]["content"].as_str().unwrap_or(""); if prompt == "retry-once" && calls .lock() .await .iter() - .filter(|call| call["messages"][0]["content"] == "retry-once") + .filter(|call| user_prompt(call) == "retry-once") .count() == 1 { @@ -142,7 +158,7 @@ async fn upstream_chat( ) .into_response(); } - if model == "model/weak" && body["messages"][0]["content"] == "overflow" { + if model == "model/weak" && prompt == "overflow" { return ( StatusCode::BAD_REQUEST, Json(json!({ @@ -157,7 +173,7 @@ async fn upstream_chat( if body["stream"].as_bool() == Some(true) { // Streamed tool call, for the namespace-on-every-event assertions. The // model calls a tool by the name it was given, so echo that name back. - if body["messages"][0]["content"] == "mcp-tool-call" { + if prompt == "mcp-tool-call" { let called = body["tool_choice"]["function"]["name"] .as_str() .or_else(|| body["tools"][0]["function"]["name"].as_str()) @@ -176,7 +192,7 @@ async fn upstream_chat( ); return Sse::new(stream).into_response(); } - if body["messages"][0]["content"] == "stream-error" { + if prompt == "stream-error" { let events = [ json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(), json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"content": "before"}}]}).to_string(), @@ -238,7 +254,7 @@ async fn upstream_chat( } // Buffered tool call, the non-streaming counterpart of the branch above. - if body["messages"][0]["content"] == "mcp-tool-call" { + if prompt == "mcp-tool-call" { let called = body["tool_choice"]["function"]["name"] .as_str() .or_else(|| body["tools"][0]["function"]["name"].as_str()) @@ -289,10 +305,9 @@ async fn upstream_chat( } else { r#"{"decision":{"target":"premium"}}"# } - } else if model == "model/classifier" - && body - .pointer("/response_format/json_schema/schema/properties/escalate") - .is_some() + } else if body + .pointer("/response_format/json_schema/schema/properties/escalate") + .is_some() { r#"{"escalate":false,"reason":"making progress"}"# } else if model == "model/classifier" && requests_schema_invalid_verdict { @@ -862,10 +877,12 @@ max_retries = 0 [targets.first] id = "{first}" llm_client = "mock" +system_prompt = "weak answer prompt" [targets.second] id = "{second}" llm_client = "mock" +system_prompt = "strong answer prompt" [routes.random] id = "{ROUTE_MODEL}" @@ -1077,6 +1094,7 @@ base_url = "{model_url}" [targets.judge] id = "model/classifier" llm_client = "judge_provider" +system_prompt = "judge target prompt" [targets.quality] id = "model/strong" @@ -1086,6 +1104,7 @@ llm_client = "model_provider" id = "model/weak" llm_client = "model_provider" extra_body = {{ service_tier = "priority" }} +system_prompt = "economy answer prompt" [routes.classify] id = "switchyard/classify" @@ -1180,6 +1199,14 @@ escalation = {{ confirmations = 1 }} ); assert_eq!(model_upstream.models().await, ["model/weak"]); assert_eq!(judge_upstream.models().await, ["model/classifier"]); + assert!(has_system_prompt( + &model_upstream.calls.lock().await[0], + "economy answer prompt" + )); + assert!(!has_system_prompt( + &judge_upstream.calls.lock().await[0], + "judge target prompt" + )); Ok(()) } @@ -1303,8 +1330,6 @@ efficient_target = "weak" picker = "efficient_first" confidence_threshold = 0.5 recent_turn_window = 3 -capable_system_prompt = "diagnose before you edit" -efficient_system_prompt = "follow the settled plan" [routes.stage.handoff_notes] escalation_note = "the previous model was stalling" @@ -1610,16 +1635,18 @@ format = "openai_chat" base_url = "{base_url}" [targets.classifier] -id = "model/classifier" +id = "model/strong" llm_client = "upstream" [targets.strong] id = "model/strong" llm_client = "upstream" +system_prompt = "strong answer prompt" [targets.weak] id = "model/weak" llm_client = "upstream" +system_prompt = "weak answer prompt" [routes.escalation] id = "switchyard/escalation" @@ -1647,7 +1674,11 @@ escalation = {{ confirmations = 1 }} ) .await?; assert_eq!(response.status, StatusCode::OK); - assert_eq!(upstream.models().await, ["model/weak", "model/classifier"]); + assert_eq!(upstream.models().await, ["model/weak", "model/strong"]); + let calls = upstream.calls.lock().await; + assert!(has_system_prompt(&calls[0], "weak answer prompt")); + assert!(!has_system_prompt(&calls[1], "strong answer prompt")); + drop(calls); let stats = send( &app, @@ -1662,7 +1693,7 @@ escalation = {{ confirmations = 1 }} assert_eq!(stats["total_prompt_tokens"], 20); assert_eq!(stats["total_completion_tokens"], 4); assert_eq!(stats["models"]["model/weak"]["calls"], 1); - assert_eq!(stats["models"]["model/classifier"]["calls"], 1); + assert_eq!(stats["models"]["model/strong"]["calls"], 1); let process_stats = send(&app, "GET", "/v1/stats", None).await?.json()?; assert_eq!(process_stats["total_requests"], 1); @@ -1786,6 +1817,7 @@ llm_client = "responses" [targets.strong] id = "real/opus" llm_client = "claude" +system_prompt = "completion instructions" [targets.other] id = "real/sonnet" @@ -1836,6 +1868,7 @@ targets = ["responses", "other", "strong"] let calls = upstream.calls.lock().await; assert_eq!(calls.len(), 3); assert_eq!(calls[0]["model"], "real/opus"); + assert!(calls[0].get("system").is_none()); assert_eq!( calls[1], json!({ @@ -2596,13 +2629,23 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted ); assert_eq!(response.json()?["model"], "model/strong"); let calls = upstream.calls.lock().await; + let candidate_calls = &calls[previous_call_count..]; assert_eq!( - calls[previous_call_count..] + candidate_calls .iter() .map(|call| call["model"].as_str().unwrap_or("")) .collect::>(), ["model/weak", "model/strong"] ); + assert!(has_system_prompt(&candidate_calls[0], "weak answer prompt")); + assert!(has_system_prompt( + &candidate_calls[1], + "strong answer prompt" + )); + assert!(!has_system_prompt( + &candidate_calls[1], + "weak answer prompt" + )); } let stats = send(&app, "GET", "/v1/stats", None).await?.json()?; @@ -3076,10 +3119,12 @@ base_url = "{base_url}" [targets.executor] id = "model/executor" llm_client = "upstream" +system_prompt = "executor answer prompt" [targets.advisor] id = "model/advisor" llm_client = "upstream" +system_prompt = "advisor target prompt" [routes.gated] id = "switchyard/advisor" @@ -3120,6 +3165,10 @@ async fn advisor_route_approve_flow_and_stats() -> TestResult { ); // Executor turn first, then the review consult. assert_eq!(upstream.models().await, ["model/executor", "model/advisor"]); + let calls = upstream.calls.lock().await; + assert!(has_system_prompt(&calls[0], "executor answer prompt")); + assert!(!has_system_prompt(&calls[1], "advisor target prompt")); + drop(calls); let stats = send(&app, "GET", "/v1/stats", None).await?.json()?; assert_eq!(stats["models"]["model/executor"]["calls"], 1); diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 966edd978..c9ed4e9a6 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -88,8 +88,20 @@ calls an upstream. |---|:---:|---|---| | `id` | Yes | — | Exact model ID sent upstream. | | `llm_client` | Yes | — | Key under `[llm_clients]`. | +| `system_prompt` | No | unset | System prompt prepended when this target serves a completion. | | `extra_body` | No | `{}` | Values merged into the upstream request when the request does not already set that key. | +Each selected or fallback target is prepared from the routed request independently. A prompt +configured for one target is therefore not carried into another target's fallback request. +Judge-only, classifier-only, and reviewer-only targets are not completion destinations and do not +receive this prompt. + +Escalation's weak target and Advisor's executor produce a candidate response while routing, so +their target prompt is applied to that call. A prompted target in either role cannot use the same +model ID as that route's judge or reviewer because those calls would otherwise be indistinguishable +at the client boundary; Switchyard rejects that configuration when it loads. +Token-count requests do not apply target system prompts. + ## `[routes.]` Every route takes the common keys below, plus the keys for its type. @@ -212,8 +224,6 @@ optional `handoff_notes` and `classifier` tables and for tuning. | `picker` | Yes | — | `efficient_first`, or `capable_first` (experimental, unbenchmarked). Tier used when the signals are not confident. | | `confidence_threshold` | Yes | — | Corroboration a decisive pick needs. In `[0, 1]`. | | `recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. | -| `capable_system_prompt` | No | unset | System prompt handed to the capable tier. | -| `efficient_system_prompt` | No | unset | System prompt handed to the efficient tier. | | `classifier.classify_trigger` | No | `every_request` | When the judge runs. See the `llm_classifier` route. `new_session` has no effect here. | | `classifier.response_format_type` | No | `json_schema` | Structured-output mode for the optional classifier judge. Use `json_object` when the classifier provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. | | `subagents` | No | unset | Nested `passthrough` or custom `llm_classifier` policy used only for delegated sub-agent work. See [Sub-Agent-Aware Routing](../routing_algorithms/subagent_routing.md). | @@ -234,8 +244,6 @@ configuration. Today a classifier sets the tier a stage router falls open to whe | `stage.efficient_target` | Yes | — | Efficient tier. | | `stage.confidence_threshold` | Yes | — | Corroboration a decisive signal needs. In `[0, 1]`. | | `stage.recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. | -| `stage.capable_system_prompt` | No | unset | System prompt handed to the capable tier. | -| `stage.efficient_system_prompt` | No | unset | System prompt handed to the efficient tier. | | `subagents` | No | unset | Nested policy used only for delegated sub-agent work. | The tier is retained per session. A deployment that sends no session ID needs diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index c91750d00..95c484721 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -185,10 +185,12 @@ api_key_env = "OPENROUTER_API_KEY" [targets.strong] id = "openai/gpt-4o" llm_client = "openrouter" +# system_prompt = "diagnose before you edit" # optional [targets.weak] id = "openai/gpt-4o-mini" llm_client = "openrouter" +# system_prompt = "follow the settled plan" # optional [routes.stage] id = "switchyard/stage" @@ -222,15 +224,6 @@ escalation_note = "the previous model was stalling; pick up the diagnosis" # only_on_wrong_signal_escalation = true # default; set false to always send ``` -### Optional: per-tier system prompts - -```toml -[routes.stage] -# ... -capable_system_prompt = "diagnose before you edit" -efficient_system_prompt = "follow the settled plan" -``` - ### Optional: LLM classifier fallback The block is optional, and omitting it is the default. With no