(MOT-4433) test(console): cover provider family failures

console/web/e2e/harness-stack.ts

@@ -71,11 +71,8 @@ export interface HarnessStack {
   finish(): Promise<PlaygroundResult>
 }
 
-interface FixtureOptions {
-  scenario: string
-}
-
 interface FixtureValues {
+  scenario: string
   stack: HarnessStack
 }
 
@@ -186,8 +183,8 @@ function armCompletion(
   })
 }
 
-export const test = base.extend<FixtureValues, FixtureOptions>({
-  scenario: ['', { scope: 'worker', option: true }],
+export const test = base.extend<FixtureValues>({
+  scenario: ['', { option: true }],
   stack: async ({ scenario }, use, testInfo) => {
     if (!scenario) throw new Error('test.use({ scenario }) is required')
     const artifactsRoot = path.resolve(

console/web/e2e/provider-family-errors.spec.ts

@@ -0,0 +1,73 @@
+import { expect, expectPassingResult, openSession, test } from './harness-stack'
+
+const cases = [
+  {
+    scenario: 'console-anthropic-messages-error',
+    family: 'anthropic messages',
+    reason: 'anthropic messages: credit balance is too low',
+  },
+  {
+    scenario: 'console-openai-chat-error',
+    family: 'openai chat completions',
+    reason: 'openai chat completions: insufficient quota',
+  },
+  {
+    scenario: 'console-openai-responses-error',
+    family: 'openai responses',
+    reason: 'openai responses: credit balance exhausted',
+  },
+] as const
+
+const recoveryMessage =
+  'Confirm the chat can continue after the provider issue is corrected.'
+
+for (const fixture of cases) {
+  test.describe(`${fixture.family} provider failure`, () => {
+    test.use({ scenario: fixture.scenario })
+
+    test('renders and captures the permanent error notice', async ({
+      page,
+      stack,
+    }, testInfo) => {
+      const failed = stack.waitForTurnCompleted()
+      await openSession(page, stack)
+      const composer = page.getByLabel('message composer')
+      await composer.pressSequentially(stack.ready.message)
+      await page.getByRole('button', { name: 'send message' }).click()
+
+      expect(await failed).toMatchObject({
+        session_id: stack.ready.session.id,
+        status: 'failed',
+      })
+      const notice = page
+        .locator(
+          '[data-message-role="system-notice"][data-message-tone="error"]',
+        )
+        .filter({ hasText: fixture.reason })
+      await expect(notice).toHaveCount(1)
+      await expect(notice).toContainText('turn failed [llm.permanent]')
+
+      const screenshot = testInfo.outputPath(`${fixture.scenario}.png`)
+      await page.screenshot({ path: screenshot, fullPage: true })
+      await testInfo.attach(`console-${fixture.scenario}`, {
+        path: screenshot,
+        contentType: 'image/png',
+      })
+
+      const recovered = stack.waitForTurnCompleted()
+      await composer.pressSequentially(recoveryMessage)
+      await page.getByRole('button', { name: 'send message' }).click()
+      expect(await recovered).toMatchObject({
+        session_id: stack.ready.session.id,
+        status: 'completed',
+      })
+      await expect(
+        page.locator('[data-message-role="assistant"]', {
+          hasText: 'provider family recovery complete',
+        }),
+      ).toHaveCount(1)
+
+      expectPassingResult(await stack.finish())
+    })
+  })
+}

console/web/src/components/chat/ChatView.tsx

@@ -1462,6 +1462,10 @@ export function ChatView({
                 kind: 'notice',
                 content: noticeContent,
                 tone: event.reason === 'error' ? 'error' : 'warn',
+                // The transcript owns the authoritative lifecycle notice under
+                // this id. Trigger delivery is unordered, so a late live
+                // fallback may fill a gap but must not overwrite that record.
+                provisional: true,
                 createdAt: Date.now(),
               }
               onAppendMessage(conversationId, notice)

console/web/src/components/chat/Message.tsx

@@ -140,6 +140,8 @@ function SystemNotice({ message }: { message: SystemMessageType }) {
         : 'border-l-rule text-ink-faint'
   return (
     <article
+      data-message-role="system-notice"
+      data-message-tone={tone}
       className={cn(
         'border-l-2 pl-3 py-1 font-mono text-[12px] uppercase tracking-[0.04em]',
         toneCls,

console/web/src/hooks/use-conversations.test.ts

@@ -305,6 +305,7 @@ describe('appendMessageToConversation', () => {
             kind: 'notice',
             tone: 'error',
             content: 'response failed',
+            provisional: true,
             createdAt: 3_000,
           },
         ],
@@ -324,6 +325,40 @@ describe('appendMessageToConversation', () => {
       id: 'e_t-1_error',
       content: 'turn failed [llm.transient] — exact reason',
     })
+    expect(next.messages[0]).not.toHaveProperty('provisional')
+  })
+
+  it('does not overwrite a durable lifecycle notice with a late live fallback', () => {
+    const next = appendMessageToConversation(
+      conversation({
+        messages: [
+          {
+            id: 'e_t-1_error',
+            role: 'system',
+            kind: 'notice',
+            tone: 'error',
+            content: 'turn failed [llm.permanent] — exact reason',
+            createdAt: 3_000,
+          },
+        ],
+      }),
+      {
+        id: 'e_t-1_error',
+        role: 'system',
+        kind: 'notice',
+        tone: 'error',
+        content: 'response failed: fallback reason',
+        provisional: true,
+        createdAt: 3_100,
+      },
+    )
+
+    expect(next.messages).toHaveLength(1)
+    expect(next.messages[0]).toMatchObject({
+      id: 'e_t-1_error',
+      content: 'turn failed [llm.permanent] — exact reason',
+    })
+    expect(next.messages[0]).not.toHaveProperty('provisional')
   })
 })
 

console/web/src/hooks/use-conversations.ts

@@ -304,12 +304,20 @@ export function appendMessageToConversation(
   now = Date.now(),
 ): Conversation {
   const existingIndex = c.messages.findIndex((item) => item.id === message.id)
+  const existing = existingIndex === -1 ? undefined : c.messages[existingIndex]
+  const preservesDurableNotice =
+    existing?.role === 'system' &&
+    message.role === 'system' &&
+    message.provisional === true &&
+    existing.provisional !== true
   const messages =
     existingIndex === -1
       ? [...c.messages, message]
-      : c.messages.map((item, index) =>
-          index === existingIndex ? message : item,
-        )
+      : preservesDurableNotice
+        ? c.messages
+        : c.messages.map((item, index) =>
+            index === existingIndex ? message : item,
+          )
   const next: Conversation = {
     ...c,
     messages,

console/web/src/types/chat.ts

@@ -199,6 +199,12 @@ export interface SystemMessage extends BaseMessage {
   content: string
   tone?: 'info' | 'warn' | 'error'
   kind?: 'notice' | 'compaction' | 'trigger-fired'
+  /**
+   * Live-only fallback for a durable transcript entry with the same id.
+   * It may fill a delivery gap, but must never replace the transcript-backed
+   * message when lifecycle and transcript events arrive out of order.
+   */
+  provisional?: boolean
   summaryText?: string
   tokensBefore?: number
   /** Present on `kind: 'trigger-fired'`. */

harness/tests/integration/README.md

@@ -31,6 +31,9 @@ No provider key or network access is required.
 | INT-021 | `router-midstream-terminal-error` | direct | partial content and keepalive noise followed by one permanent router error preserve the partial, fail exactly once, and leave no pending work |
 | UI-001 | `console-streamed-text` | playground | a message sent by the Console streams to durable completion |
 | UI-002 | `multi-turn-traces` | playground | a native function turn and a Console turn expose distinct traces and function-call events |
+| UI-003 | `console-anthropic-messages-error` | playground | an Anthropic Messages permanent provider failure is shown and the chat recovers |
+| UI-004 | `console-openai-chat-error` | playground | an OpenAI Chat Completions permanent provider failure is shown and the chat recovers |
+| UI-005 | `console-openai-responses-error` | playground | an OpenAI Responses permanent provider failure is shown and the chat recovers |
 
 Each fixture is defined end to end in its own `src/scenarios/*.rs` file with a
 small typed DSL. The scenario keeps its send policy, router request matchers,
@@ -112,8 +115,8 @@ cargo clippy --manifest-path harness/Cargo.toml \
 ```
 
 `validate --scenario all` checks every fixture. `run --scenario all` executes
-all direct scenarios; UI-001 and UI-002 must use `playground`. INT-003 produces
-two terminal turns from one send: generation 1
+all direct scenarios; UI-001 through UI-005 must use `playground`. INT-003
+produces two terminal turns from one send: generation 1
 steers a message into the running session (it parks durably) and then fails,
 so the harness's failed finalize drains the parked row and reseeds a turn to
 react to it. The failed route is deliberate — a park during a *completing*

harness/tests/integration/src/fixtures/tests.rs

@@ -12,7 +12,7 @@ fn all_selection_returns_the_checked_in_fixtures() {
         std::collections::BTreeSet::from([
             "INT-001", "INT-002", "INT-003", "INT-005", "INT-006", "INT-010", "INT-011", "INT-012",
             "INT-013", "INT-014", "INT-015", "INT-016", "INT-017", "INT-018", "INT-019", "INT-020",
-            "INT-021", "UI-001", "UI-002"
+            "INT-021", "UI-001", "UI-002", "UI-003", "UI-004", "UI-005"
         ])
     );
     assert_eq!(

harness/tests/integration/src/scenario/playground.rs

@@ -23,7 +23,12 @@ use super::runner::{BootedRun, ExpandedRun, ScenarioRunner};
 use super::state::{ActiveTurn, PreparedRun};
 
 const CONSOLE_CONNECT_INTERVAL: Duration = Duration::from_millis(100);
-const SHUTDOWN_COMPLETION_GRACE: Duration = Duration::from_secs(1);
+// Console and evidence subscribers receive the same completion concurrently.
+// Under CI load, Playwright can observe the terminal turn and request shutdown
+// before the probe has drained its delivery. Keep shutdown graceful long
+// enough for that already-emitted event without relaxing the scenario deadline
+// or the required completion count.
+const SHUTDOWN_COMPLETION_GRACE: Duration = Duration::from_secs(5);
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]

harness/tests/integration/src/scenarios/mod.rs

@@ -10,6 +10,7 @@ mod engine_restart_recovery;
 mod exactly_once_function;
 mod leaf_denied_control_plane;
 mod multi_turn_traces;
+mod provider_family_errors;
 mod queued_message_edit_unqueue;
 mod reseed_parked_message;
 mod router_midstream_terminal_error;
@@ -35,7 +36,7 @@ pub enum ScenarioDriver {
 
 /// Every fixture, in stable slug order.
 pub fn all() -> Vec<ScenarioFixture> {
-    vec![
+    let mut fixtures = vec![
         child_discovery_granted::scenario(),
         condition_failure_notice::scenario(),
         console_streamed_text::scenario(),
@@ -55,7 +56,9 @@ pub fn all() -> Vec<ScenarioFixture> {
         streamed_text::scenario(),
         wake_expiry_notice::scenario(),
         timer_wake::scenario(),
-    ]
+    ];
+    fixtures.extend(provider_family_errors::scenarios());
+    fixtures
 }
 
 #[cfg(test)]
@@ -65,7 +68,7 @@ mod tests {
     #[test]
     fn every_fixture_is_unique_and_valid() {
         let fixtures = all();
-        assert_eq!(fixtures.len(), 19);
+        assert_eq!(fixtures.len(), 22);
         let mut slugs = std::collections::BTreeSet::new();
         let mut ids = std::collections::BTreeSet::new();
         for fixture in fixtures {