Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions crates/libsy/src/algorithms/advisor_gate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ use switchyard_protocol::{
SamplingParams,
};

use crate::algorithms::util::tool_signals::ToolSignals;
use crate::core::algorithm::{Algorithm, Driver, RoutingOutcome};
use crate::{LibsyError, Result};

Expand All @@ -50,10 +51,7 @@ use telemetry::{
record_review,
};
use transcript::{VERDICT_PATTERN, Verdict, advisor_reply_text, parse_verdict, review_transcript};
use turn::{
GatedTurn, assistant_turns, buffer_turn, count_tool_results, has_tool_use, reasoning_text,
visible_text,
};
use turn::{GatedTurn, buffer_turn, has_tool_use, reasoning_text, visible_text};

/// APPROVE/REDO reviewer contract sent as the advisor's system prompt.
pub const REVIEWER_SYSTEM_PROMPT: &str =
Expand Down Expand Up @@ -338,21 +336,24 @@ impl AdvisorGate {
.await?;
let turn = buffer_turn(self.executor.as_str(), response).await?;

// Request-side guard counts come from the shared ToolSignals
// extraction — the same definition of tool-result/turn counting the
// stage router uses.
let signals = ToolSignals::from_request(&request, None);
// The stall checkpoint fires once per conversation regardless of the
// turn's shape — even a tool-call turn — for executors that grind
// without ever declaring completion.
let stall_key = stall_key(&request);
let stall = self.config.gate_stall_turns > 0
&& !self.stall_already_fired(stall_key)
&& assistant_turns(&request.llm_request.messages) >= self.config.gate_stall_turns;
&& signals.assistant_turn_count >= self.config.gate_stall_turns;
let triggered = match &self.trigger {
CompiledTrigger::Pattern(pattern) => {
pattern.is_match(visible_text(&turn.agg).as_deref().unwrap_or(""))
}
CompiledTrigger::NoToolCall => {
!has_tool_use(&turn.agg)
&& count_tool_results(&request.llm_request.messages)
>= self.config.gate_min_tool_results
&& signals.tool_result_count >= self.config.gate_min_tool_results
}
};
if !(triggered || stall) {
Expand Down
53 changes: 53 additions & 0 deletions crates/libsy/src/algorithms/advisor_gate/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -795,6 +795,36 @@ async fn min_tool_results_defers_gate() {
assert_eq!(script.advisor_consults(), 1);
}

#[tokio::test]
async fn min_tool_results_counts_batched_result_blocks() {
// Anthropic batches several tool_result blocks into one user message;
// the guard counts blocks, not messages.
let script = Script::new();
let gate = gate(AdvisorGateConfig {
gate_min_tool_results: 2,
..AdvisorGateConfig::default()
});
let tool_result = |id: &str| {
ContentBlock::ToolResult(ToolResult {
tool_call_id: id.to_string(),
content: vec![ContentBlock::Text {
text: "ok".to_string(),
}],
is_error: None,
})
};
let batched = request(vec![
Message::text(Role::User, "build X"),
Message {
role: Role::User,
content: vec![tool_result("t1"), tool_result("t2")],
},
]);
let serve = script.serve("APPROVE", |_| reply("done"));
test_drive(gate, batched, serve).await.expect("routes");
assert_eq!(script.advisor_consults(), 1);
}

#[tokio::test]
async fn stall_checkpoint_reviews_mid_task_once() {
let script = Script::new();
Expand Down Expand Up @@ -860,6 +890,29 @@ async fn simultaneous_trigger_does_not_latch_stall() {
assert_eq!(script.advisor_consults(), 2);
}

#[tokio::test]
async fn stall_counts_assistant_turns_not_messages() {
// Three messages but one assistant turn: below the threshold, so the
// checkpoint stays quiet — the stall clock is assistant turns, not
// conversation length.
let script = Script::new();
let gate = gate(AdvisorGateConfig {
gate_stall_turns: 2,
..AdvisorGateConfig::default()
});
let conversation = request(vec![
Message::text(Role::User, "build X"),
Message::text(Role::Assistant, "step 1"),
Message::text(Role::User, "keep going"),
]);
let serve = script.serve("APPROVE", {
let turn = parking_lot::Mutex::new(Some(tool_call_turn()));
move |_| turn.lock().take().expect("one executor call")
});
test_drive(gate, conversation, serve).await.expect("routes");
assert_eq!(script.advisor_consults(), 0);
}

// ── Reasoning-only and empty turns ──────────────────────────────────────

#[tokio::test]
Expand Down
22 changes: 1 addition & 21 deletions crates/libsy/src/algorithms/advisor_gate/turn.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
use futures::StreamExt;
use switchyard_protocol::{
AggLlmResponse, ContentBlock, LlmClientError, LlmResponse, LlmResponseChunk,
LlmResponseStreamEvent, Message, Metadata, Response, ResponseAccumulator, Role, StopReason,
LlmResponseStreamEvent, Metadata, Response, ResponseAccumulator, StopReason,
};

use crate::{LibsyError, Result};
Expand Down Expand Up @@ -151,23 +151,3 @@ pub(super) fn reasoning_text(agg: &AggLlmResponse) -> Option<String> {
Some(joined)
}
}

/// Tool results carried by the conversation so far (both wires normalize
/// tool results into `ContentBlock::ToolResult`).
pub(super) fn count_tool_results(messages: &[Message]) -> u32 {
let count = messages
.iter()
.flat_map(|message| message.content.iter())
.filter(|block| matches!(block, ContentBlock::ToolResult(_)))
.count();
u32::try_from(count).unwrap_or(u32::MAX)
}

/// Assistant turns already in the request — the stall checkpoint's clock.
pub(super) fn assistant_turns(messages: &[Message]) -> u32 {
let count = messages
.iter()
.filter(|message| message.role == Role::Assistant)
.count();
u32::try_from(count).unwrap_or(u32::MAX)
}
61 changes: 57 additions & 4 deletions crates/libsy/src/algorithms/util/tool_signals.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,16 @@
//!
//! The extractor walks normalized messages, finds tool calls and results,
//! pattern-matches their text against a curated error table, and aggregates
//! conversation-history metrics used by [`crate::StageRouter`].
//! conversation-history metrics used by [`crate::StageRouter`] and the
//! advisor gate's request-side guards.
//!
//! All logic is pure and deterministic — no I/O, no shared state.

#![allow(dead_code)]

use async_trait::async_trait;
use serde_json::Value;
use switchyard_protocol::{ContentBlock, Request};
use switchyard_protocol::{ContentBlock, Request, Role};

use crate::Result;

Expand Down Expand Up @@ -207,7 +208,9 @@ pub const DEFAULT_RECENT_WINDOW: usize = 3;
/// Tool-execution signals extracted from a normalized [`Request`].
///
/// A request-side processor stores these signals in [`State`](crate::State) for
/// [`crate::StageRouter`] and its classifier to consume.
/// [`crate::StageRouter`] and its classifier to consume. The advisor gate's
/// request-side guards read the conversation-shape counts directly via
/// [`ToolSignals::from_request`].
#[derive(Clone, Debug, Default)]
pub struct ToolSignals {
/// Max severity across the recent window (last `recent_window` tool results):
Expand Down Expand Up @@ -239,6 +242,12 @@ pub struct ToolSignals {
pub pure_bash_streak: u32,
/// At least one of the last three tool results matched a test-pass pattern.
pub tests_passed: bool,
/// Total `ToolResult` blocks, counted per block (a message batching N
/// results contributes N) and including empty-content results.
pub tool_result_count: u32,
/// Messages with `Role::Assistant`, unlike [`ToolSignals::turn_depth`],
/// which counts every message regardless of role.
pub assistant_turn_count: u32,
/// Message-count proxy for turn depth. Wire-format dependent (Anthropic batches
/// tool results into fewer messages than OpenAI-chat), so gates keyed on it are
/// approximate across request origins.
Expand Down Expand Up @@ -353,8 +362,13 @@ fn extract_tool_signals_with_window(request: &Request, recent_window: usize) ->
let mut tool_texts: Vec<String> = Vec::new();
let mut tool_calls: Vec<ObservedToolCall> = Vec::new();
let mut compacted = false;
let mut tool_result_count = 0usize;
let mut assistant_turn_count = 0usize;

for message in messages {
if message.role == Role::Assistant {
assistant_turn_count += 1;
}
for block in &message.content {
match block {
ContentBlock::ToolCall(call) => {
Expand All @@ -364,6 +378,8 @@ fn extract_tool_signals_with_window(request: &Request, recent_window: usize) ->
});
}
ContentBlock::ToolResult(result) => {
// Before the empty-text filter: empty results still count.
tool_result_count += 1;
let text = result
.content
.iter()
Expand All @@ -387,6 +403,8 @@ fn extract_tool_signals_with_window(request: &Request, recent_window: usize) ->

let mut signal = build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window);
signal.compacted = compacted;
signal.tool_result_count = u32::try_from(tool_result_count).unwrap_or(u32::MAX);
signal.assistant_turn_count = u32::try_from(assistant_turn_count).unwrap_or(u32::MAX);
signal
}

Expand Down Expand Up @@ -524,7 +542,10 @@ fn build_signal(
tests_passed,
turn_depth,
// Set by extract_tool_signals_with_window after the format-specific extract,
// which scans all message contents for the compaction marker.
// which scans all message contents for the compaction marker and tallies
// the raw conversation-shape counts.
tool_result_count: 0,
assistant_turn_count: 0,
compacted: false,
}
}
Expand Down Expand Up @@ -803,6 +824,38 @@ mod tests {
assert_eq!(sig.write_count, 1);
}

#[test]
fn conversation_counts_are_per_block_and_role_aware() {
// A batched user message (Anthropic shape) contributes one count per
// ToolResult block, empty-content results included. assistant_turn_count
// tracks Role::Assistant only, while turn_depth counts every message.
let result = |content: Vec<ContentBlock>| {
ContentBlock::ToolResult(ToolResult {
tool_call_id: String::new(),
content,
is_error: None,
})
};
let request = with_messages(vec![
Message::text(Role::User, "do something"),
Message::text(Role::Assistant, "working"),
Message {
role: Role::User,
content: vec![
result(vec![ContentBlock::Text {
text: "ok".to_string(),
}]),
result(Vec::new()),
],
},
tc("Bash"),
]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.tool_result_count, 2);
assert_eq!(sig.assistant_turn_count, 2);
assert_eq!(sig.turn_depth, 4);
}

#[test]
fn recent_window_counts_only_last_default_window_tool_calls() {
// 5 writes + 1 edit at the end → the default window (3) should see
Expand Down
Loading