From 638dc4af3198d8131e5f1785b7b07a36b62e6594 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Fri, 14 Aug 2026 09:59:07 -0300 Subject: [PATCH 1/3] (MOT-4433) test(console): cover provider family failures --- console/web/e2e/harness-stack.ts | 9 +- .../web/e2e/provider-family-errors.spec.ts | 73 ++++++++ console/web/src/components/chat/Message.tsx | 2 + harness/tests/integration/README.md | 4 +- .../tests/integration/src/fixtures/tests.rs | 2 +- .../tests/integration/src/scenarios/mod.rs | 9 +- .../src/scenarios/provider_family_errors.rs | 171 ++++++++++++++++++ 7 files changed, 258 insertions(+), 12 deletions(-) create mode 100644 console/web/e2e/provider-family-errors.spec.ts create mode 100644 harness/tests/integration/src/scenarios/provider_family_errors.rs diff --git a/console/web/e2e/harness-stack.ts b/console/web/e2e/harness-stack.ts index 7cec043e3..d378183ef 100644 --- a/console/web/e2e/harness-stack.ts +++ b/console/web/e2e/harness-stack.ts @@ -71,11 +71,8 @@ export interface HarnessStack { finish(): Promise } -interface FixtureOptions { - scenario: string -} - interface FixtureValues { + scenario: string stack: HarnessStack } @@ -186,8 +183,8 @@ function armCompletion( }) } -export const test = base.extend({ - scenario: ['', { scope: 'worker', option: true }], +export const test = base.extend({ + scenario: ['', { option: true }], stack: async ({ scenario }, use, testInfo) => { if (!scenario) throw new Error('test.use({ scenario }) is required') const artifactsRoot = path.resolve( diff --git a/console/web/e2e/provider-family-errors.spec.ts b/console/web/e2e/provider-family-errors.spec.ts new file mode 100644 index 000000000..140bfd156 --- /dev/null +++ b/console/web/e2e/provider-family-errors.spec.ts @@ -0,0 +1,73 @@ +import { expect, expectPassingResult, openSession, test } from './harness-stack' + +const cases = [ + { + scenario: 'console-anthropic-messages-error', + family: 'anthropic messages', + reason: 'anthropic messages: credit balance is too low', + }, + { + scenario: 'console-openai-chat-error', + family: 'openai chat completions', + reason: 'openai chat completions: insufficient quota', + }, + { + scenario: 'console-openai-responses-error', + family: 'openai responses', + reason: 'openai responses: credit balance exhausted', + }, +] as const + +const recoveryMessage = + 'Confirm the chat can continue after the provider issue is corrected.' + +for (const fixture of cases) { + test.describe(`${fixture.family} provider failure`, () => { + test.use({ scenario: fixture.scenario }) + + test('renders and captures the permanent error notice', async ({ + page, + stack, + }, testInfo) => { + const failed = stack.waitForTurnCompleted() + await openSession(page, stack) + const composer = page.getByLabel('message composer') + await composer.pressSequentially(stack.ready.message) + await page.getByRole('button', { name: 'send message' }).click() + + expect(await failed).toMatchObject({ + session_id: stack.ready.session.id, + status: 'failed', + }) + const notice = page + .locator( + '[data-message-role="system-notice"][data-message-tone="error"]', + ) + .filter({ hasText: fixture.reason }) + await expect(notice).toHaveCount(1) + await expect(notice).toContainText('turn failed [llm.permanent]') + + const screenshot = testInfo.outputPath(`${fixture.scenario}.png`) + await page.screenshot({ path: screenshot, fullPage: true }) + await testInfo.attach(`console-${fixture.scenario}`, { + path: screenshot, + contentType: 'image/png', + }) + + const recovered = stack.waitForTurnCompleted() + await composer.pressSequentially(recoveryMessage) + await page.getByRole('button', { name: 'send message' }).click() + expect(await recovered).toMatchObject({ + session_id: stack.ready.session.id, + status: 'completed', + }) + await expect( + page.locator('[data-message-role="assistant"]', { + hasText: 'provider family recovery complete', + }), + ).toHaveCount(1) + + expectPassingResult(await stack.finish()) + }) + }) +} diff --git a/console/web/src/components/chat/Message.tsx b/console/web/src/components/chat/Message.tsx index e2021540a..c30083823 100644 --- a/console/web/src/components/chat/Message.tsx +++ b/console/web/src/components/chat/Message.tsx @@ -140,6 +140,8 @@ function SystemNotice({ message }: { message: SystemMessageType }) { : 'border-l-rule text-ink-faint' return (
Vec { - vec![ + let mut fixtures = vec![ child_discovery_granted::scenario(), condition_failure_notice::scenario(), console_streamed_text::scenario(), @@ -53,7 +54,9 @@ pub fn all() -> Vec { streamed_text::scenario(), wake_expiry_notice::scenario(), timer_wake::scenario(), - ] + ]; + fixtures.extend(provider_family_errors::scenarios()); + fixtures } #[cfg(test)] @@ -63,7 +66,7 @@ mod tests { #[test] fn every_fixture_is_unique_and_valid() { let fixtures = all(); - assert_eq!(fixtures.len(), 18); + assert_eq!(fixtures.len(), 21); let mut slugs = std::collections::BTreeSet::new(); let mut ids = std::collections::BTreeSet::new(); for fixture in fixtures { diff --git a/harness/tests/integration/src/scenarios/provider_family_errors.rs b/harness/tests/integration/src/scenarios/provider_family_errors.rs new file mode 100644 index 000000000..f85e8f2b0 --- /dev/null +++ b/harness/tests/integration/src/scenarios/provider_family_errors.rs @@ -0,0 +1,171 @@ +//! UI-003..005 — representative provider-protocol failures stay actionable +//! through the Harness and Console boundary. +//! +//! Provider-specific request and error parsing lives in the hermetic provider +//! contract suite. These fixtures begin at the normalized router boundary and +//! pin the user-facing behavior shared by each protocol family: a permanent +//! generation failure finalizes the turn, persists structured failure data, +//! and remains visible after Console transcript reconciliation. + +use serde_json::Value; + +use super::dsl::{Generation, Message, Model, Request, Response, Scenario, Send, Tool}; +use super::{ScenarioDriver, VerifyFn}; +use crate::evidence_data::RunEvidence; +use crate::fixtures::ScenarioFixture; + +const ANTHROPIC_REASON: &str = "anthropic messages: credit balance is too low"; +const CHAT_REASON: &str = "openai chat completions: insufficient quota"; +const RESPONSES_REASON: &str = "openai responses: credit balance exhausted"; +const RECOVERY_MESSAGE: &str = + "Confirm the chat can continue after the provider issue is corrected."; +const RECOVERY_TEXT: &str = "provider family recovery complete"; + +struct FamilyCase { + id: &'static str, + slug: &'static str, + model: &'static str, + reason: &'static str, + verify: VerifyFn, +} + +pub(super) fn scenarios() -> Vec { + [ + FamilyCase { + id: "UI-003", + slug: "console-anthropic-messages-error", + model: "anthropic-messages-fixture", + reason: ANTHROPIC_REASON, + verify: verify_anthropic, + }, + FamilyCase { + id: "UI-004", + slug: "console-openai-chat-error", + model: "openai-chat-completions-fixture", + reason: CHAT_REASON, + verify: verify_chat, + }, + FamilyCase { + id: "UI-005", + slug: "console-openai-responses-error", + model: "openai-responses-fixture", + reason: RESPONSES_REASON, + verify: verify_responses, + }, + ] + .into_iter() + .map(scenario) + .collect() +} + +fn scenario(case: FamilyCase) -> ScenarioFixture { + let message = format!("Exercise the {} failure path.", case.model); + let model = Model::scripted(case.model); + Scenario::new( + case.id, + case.slug, + "A permanent provider failure is persisted and shown as an actionable Console notice.", + ScenarioDriver::Playground, + model.clone(), + ) + .send( + Send::message(&message) + .idempotency_key(&format!("{{{{run_id}}}}:{}", case.slug)) + .without_functions(), + ) + .terminal_turn_statuses(["failed", "completed"]) + .generation( + Generation::new(1) + .expect( + Request::new() + .turn_request() + .system_prompt_regex("agent_trigger") + .messages_exact([Message::user(&message)]) + .tools_subset([Tool::named("agent_trigger")]), + ) + .fails(case.reason), + ) + .generation( + Generation::new(2) + .expect( + Request::new() + .turn_request_step(0) + .system_prompt_regex("agent_trigger") + .messages_exact([ + Message::user(&message), + Message::assistant_empty(&model), + Message::user(RECOVERY_MESSAGE), + ]) + .tools_subset([Tool::named("agent_trigger")]), + ) + .respond(Response::text(RECOVERY_TEXT, 12, 4)), + ) + .verify(case.verify) + .build() +} + +fn verify_anthropic(run: &RunEvidence) -> anyhow::Result<()> { + verify_permanent_failure(run, ANTHROPIC_REASON) +} + +fn verify_chat(run: &RunEvidence) -> anyhow::Result<()> { + verify_permanent_failure(run, CHAT_REASON) +} + +fn verify_responses(run: &RunEvidence) -> anyhow::Result<()> { + verify_permanent_failure(run, RESPONSES_REASON) +} + +fn verify_permanent_failure(run: &RunEvidence, expected_reason: &str) -> anyhow::Result<()> { + run.expect_assistant_texts([RECOVERY_TEXT])?; + run.expect_message_counts(2, 2, 0)?; + run.expect_no_duplicate_messages()?; + + let error = run + .transcript + .iter() + .filter_map(|item| item.get("custom")) + .find(|custom| custom.get("custom_type").and_then(Value::as_str) == Some("error")) + .ok_or_else(|| anyhow::anyhow!("durable error record is missing"))?; + let data = error.get("data").cloned().unwrap_or(Value::Null); + anyhow::ensure!( + data.get("code").and_then(Value::as_str) == Some("llm.permanent"), + "failure code is not permanent: {data}" + ); + anyhow::ensure!( + data.get("retryable").and_then(Value::as_bool) == Some(false), + "permanent failure is marked retryable: {data}" + ); + anyhow::ensure!( + data.get("phase").and_then(Value::as_str) == Some("generation"), + "failure phase is not generation: {data}" + ); + anyhow::ensure!( + data.get("summary") + .and_then(Value::as_str) + .is_some_and(|summary| summary.contains(expected_reason)), + "failure summary does not preserve the provider reason: {data}" + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn covers_each_protocol_family_with_a_permanent_terminal_failure() { + let fixtures = scenarios(); + assert_eq!(fixtures.len(), 3); + for fixture in fixtures { + fixture.validate().unwrap(); + assert_eq!(fixture.expected_turn_statuses, ["failed", "completed"]); + assert_eq!(fixture.script.generations.len(), 2); + let failed = &fixture.script.generations[0]; + assert!(failed.failure.is_some()); + assert!(failed.frames.is_empty()); + assert!(!failed.response.ok); + assert!(fixture.script.generations[1].response.ok); + } + } +} From 5e51b11441704eaa03331c24df6f30928aa94021 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Fri, 14 Aug 2026 10:33:08 -0300 Subject: [PATCH 2/3] (MOT-4433) test(harness): stabilize playground shutdown --- harness/tests/integration/src/scenario/playground.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/harness/tests/integration/src/scenario/playground.rs b/harness/tests/integration/src/scenario/playground.rs index b6d12929e..9d05d5772 100644 --- a/harness/tests/integration/src/scenario/playground.rs +++ b/harness/tests/integration/src/scenario/playground.rs @@ -23,7 +23,12 @@ use super::runner::{BootedRun, ExpandedRun, ScenarioRunner}; use super::state::{ActiveTurn, PreparedRun}; const CONSOLE_CONNECT_INTERVAL: Duration = Duration::from_millis(100); -const SHUTDOWN_COMPLETION_GRACE: Duration = Duration::from_secs(1); +// Console and evidence subscribers receive the same completion concurrently. +// Under CI load, Playwright can observe the terminal turn and request shutdown +// before the probe has drained its delivery. Keep shutdown graceful long +// enough for that already-emitted event without relaxing the scenario deadline +// or the required completion count. +const SHUTDOWN_COMPLETION_GRACE: Duration = Duration::from_secs(5); #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] From 20d78cfc4912c6c8893e1b8a5b3e95c101b00ee8 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Fri, 14 Aug 2026 21:13:41 -0300 Subject: [PATCH 3/3] (MOT-4433) fix(console): preserve durable lifecycle notices --- console/web/src/components/chat/ChatView.tsx | 4 +++ .../web/src/hooks/use-conversations.test.ts | 35 +++++++++++++++++++ console/web/src/hooks/use-conversations.ts | 14 ++++++-- console/web/src/types/chat.ts | 6 ++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/console/web/src/components/chat/ChatView.tsx b/console/web/src/components/chat/ChatView.tsx index 93481bd0f..b1ef63d4f 100644 --- a/console/web/src/components/chat/ChatView.tsx +++ b/console/web/src/components/chat/ChatView.tsx @@ -1462,6 +1462,10 @@ export function ChatView({ kind: 'notice', content: noticeContent, tone: event.reason === 'error' ? 'error' : 'warn', + // The transcript owns the authoritative lifecycle notice under + // this id. Trigger delivery is unordered, so a late live + // fallback may fill a gap but must not overwrite that record. + provisional: true, createdAt: Date.now(), } onAppendMessage(conversationId, notice) diff --git a/console/web/src/hooks/use-conversations.test.ts b/console/web/src/hooks/use-conversations.test.ts index 145e341fd..f197858c5 100644 --- a/console/web/src/hooks/use-conversations.test.ts +++ b/console/web/src/hooks/use-conversations.test.ts @@ -305,6 +305,7 @@ describe('appendMessageToConversation', () => { kind: 'notice', tone: 'error', content: 'response failed', + provisional: true, createdAt: 3_000, }, ], @@ -324,6 +325,40 @@ describe('appendMessageToConversation', () => { id: 'e_t-1_error', content: 'turn failed [llm.transient] — exact reason', }) + expect(next.messages[0]).not.toHaveProperty('provisional') + }) + + it('does not overwrite a durable lifecycle notice with a late live fallback', () => { + const next = appendMessageToConversation( + conversation({ + messages: [ + { + id: 'e_t-1_error', + role: 'system', + kind: 'notice', + tone: 'error', + content: 'turn failed [llm.permanent] — exact reason', + createdAt: 3_000, + }, + ], + }), + { + id: 'e_t-1_error', + role: 'system', + kind: 'notice', + tone: 'error', + content: 'response failed: fallback reason', + provisional: true, + createdAt: 3_100, + }, + ) + + expect(next.messages).toHaveLength(1) + expect(next.messages[0]).toMatchObject({ + id: 'e_t-1_error', + content: 'turn failed [llm.permanent] — exact reason', + }) + expect(next.messages[0]).not.toHaveProperty('provisional') }) }) diff --git a/console/web/src/hooks/use-conversations.ts b/console/web/src/hooks/use-conversations.ts index eacbab260..20b969059 100644 --- a/console/web/src/hooks/use-conversations.ts +++ b/console/web/src/hooks/use-conversations.ts @@ -304,12 +304,20 @@ export function appendMessageToConversation( now = Date.now(), ): Conversation { const existingIndex = c.messages.findIndex((item) => item.id === message.id) + const existing = existingIndex === -1 ? undefined : c.messages[existingIndex] + const preservesDurableNotice = + existing?.role === 'system' && + message.role === 'system' && + message.provisional === true && + existing.provisional !== true const messages = existingIndex === -1 ? [...c.messages, message] - : c.messages.map((item, index) => - index === existingIndex ? message : item, - ) + : preservesDurableNotice + ? c.messages + : c.messages.map((item, index) => + index === existingIndex ? message : item, + ) const next: Conversation = { ...c, messages, diff --git a/console/web/src/types/chat.ts b/console/web/src/types/chat.ts index a6e834694..2890729de 100644 --- a/console/web/src/types/chat.ts +++ b/console/web/src/types/chat.ts @@ -199,6 +199,12 @@ export interface SystemMessage extends BaseMessage { content: string tone?: 'info' | 'warn' | 'error' kind?: 'notice' | 'compaction' | 'trigger-fired' + /** + * Live-only fallback for a durable transcript entry with the same id. + * It may fill a delivery gap, but must never replace the transcript-backed + * message when lifecycle and transcript events arrive out of order. + */ + provisional?: boolean summaryText?: string tokensBefore?: number /** Present on `kind: 'trigger-fired'`. */