fix: fail over unhealthy Daytona sandbox hosts (#183)

## Summary

- Detect Daytona's explicit host-recovery start rejection at the
provider boundary.
- Replace only the stopped canonical runtime while retaining the user's
mounted workspace-volume subpath.
- Extend durable preparation retries so remaining transient provider
failures do not become immediate failed runs.

## Architecture

A run still resolves the single canonical user sandbox first. If Daytona
reports that the sandbox's assigned host is recovering, the existing
runtime-replacement fence verifies the active-run lease and canonical
volume identity, removes the disposable container, and creates one
replacement attached to the same isolated volume subpath. Cloudflare
Workflow retries remain the durable fallback around preparation.

## Decisions Made

| Decision | Choice | Alternatives considered | Reasoning |
| --- | --- | --- | --- |
| Host-local recovery | Replace the disposable runtime | Wait
indefinitely; fail the run | The workspace is on a separate persistent
volume, so failover preserves user files and avoids dependence on one
unhealthy runner. |
| Failure classification | Match Daytona's explicit 503 host-recovery
rejection | Replace on every 503 | A narrow provider-boundary classifier
prevents destructive failover during platform-wide outages. |
| Remaining transient failures | Six exponential-backoff preparation
retries | Tight polling inside the Worker | Durable retries do not hold
an isolate or hammer Daytona. |

## Edge Cases Handled

| Scenario | Handling |
| --- | --- |
| Another run owns the sandbox | Existing lease fence refuses
replacement. |
| Sandbox identity or volume mount is ambiguous | Existing
canonical-runtime assertions fail closed. |
| Daytona changes or omits the host-recovery message | No forced
replacement; durable preparation retries apply. |
| Replacement creation is temporarily unavailable | The Workflow retries
the idempotent preparation step. |

## Verification

- [x] `pnpm lint`
- [x] `pnpm typecheck`
- [x] `pnpm turbo build --force`
- [x] `pnpm deadcode`
- [x] `pnpm architecture:check`
- [x] `pnpm turbo skills:build`
- [ ] Resume the failed production Pomodoro run after deployment and
verify the replacement sandbox preserves the project and serves the
generated preview.

Author: iamjr15

apps/agent-worker/README.md

@@ -81,6 +81,11 @@ provider keys and sandbox capabilities are reacquired inside the active step and
 Workflow storage. A Worker isolate or Durable Object eviction therefore resumes from the last
 completed step instead of losing an in-memory coroutine. Transcript publication uses deterministic
 event keys and an atomic SQLite receipt, so Workflow step replay cannot duplicate visible parts.
+Preparation uses a multi-minute durable exponential-backoff window for transient provider
+failures. Daytona's explicit host-recovery start rejection is treated as a runtime failover signal:
+after the active-run lease and canonical volume mount are verified, the stopped container is
+replaced on the same isolated workspace-volume subpath. This preserves user files while avoiding
+an indefinite dependency on one unhealthy runner.
 There is no application step, token, duration, or cost ceiling; semantic completion ends the loop,
 while per-operation timeouts and the platform Workflow limit remain operational safeguards.
 The Worker pins Cloudflare's paid-plan maximum subrequest allowance because external provider,

apps/agent-worker/src/durable-objects/agent-run-workflow.ts

@@ -47,7 +47,10 @@ import {
   WorkflowToolStepResultSchema,
 } from "./agent-run-workflow-runtime";
 
-const PREPARE_STEP = stepConfig("10 minutes", 3);
+// A stopped Daytona sandbox can temporarily reject starts while its host recovers.
+// Keep that provider recovery inside the durable preparation step so a transient
+// host event does not become a user-visible failed run.
+const PREPARE_STEP = stepConfig("10 minutes", 6);
 const MODEL_STEP = stepConfig("5 minutes", 3);
 const TOOL_STEP = stepConfig("15 minutes", 2);
 const STATE_STEP = stepConfig("2 minutes", 5);

apps/agent-worker/src/durable-objects/project-sandbox-runtime-handle.ts

@@ -1,4 +1,8 @@
-import { DaytonaClient, type DaytonaSandbox } from "@cheatcode/agent-core/tools/code";
+import {
+  DaytonaClient,
+  type DaytonaSandbox,
+  isDaytonaHostRecoveryStartError,
+} from "@cheatcode/agent-core/tools/code";
 import { previewHostnameForWorker, resolveWorkerSecret } from "@cheatcode/env";
 import { APIError, createLogger } from "@cheatcode/observability";
 import { performAccountDeletion } from "./project-sandbox-account-deletion";
@@ -59,6 +63,8 @@ interface RuntimeState {
   workspaceState: ProjectSandboxWorkspaceState | undefined;
 }
 
+type RuntimeReplacementReason = "configuration_changed" | "daytona_host_recovery";
+
 interface SandboxLeaseRuntime {
   withCleanupSignal: <Result>(operation: () => Promise<Result>) => Promise<Result | undefined>;
   withOwnerRegistration: <Result>(
@@ -265,11 +271,21 @@ async function ensureSandbox(state: RuntimeState, startingRunId?: string): Promi
   if (state.cache.sandboxId && Date.now() - state.cache.startedVerifiedAtMs < STARTED_REVERIFY_MS) {
     return state.cache.sandboxId;
   }
-  const resolved = await inspectSandbox(state);
-  if (typeof resolved === "string") {
-    return resolved;
+  try {
+    const resolved = await inspectSandbox(state);
+    if (typeof resolved === "string") {
+      return resolved;
+    }
+    return replaceSandboxRuntime(state, startingRunId, "configuration_changed");
+  } catch (error) {
+    if (!isDaytonaHostRecoveryStartError(error)) {
+      throw error;
+    }
+    createLogger().warn("sandbox_host_recovery_failover_started", {
+      sandboxId: state.identity.sandboxName(),
+    });
+    return replaceSandboxRuntime(state, startingRunId, "daytona_host_recovery");
   }
-  return replaceSandboxRuntime(state, startingRunId);
 }
 
 async function restartSandboxForWorkspaceRecovery(
@@ -376,7 +392,11 @@ async function activateResolvedSandbox(
   return resolved.id;
 }
 
-async function replaceSandboxRuntime(state: RuntimeState, startingRunId?: string): Promise<string> {
+async function replaceSandboxRuntime(
+  state: RuntimeState,
+  startingRunId: string | undefined,
+  reason: RuntimeReplacementReason,
+): Promise<string> {
   if (state.isSandboxRuntimeUpdateInProgress) {
     throw sandboxRuntimeUpdatePending(state.env.DAYTONA_SANDBOX_SNAPSHOT);
   }
@@ -391,8 +411,8 @@ async function replaceSandboxRuntime(state: RuntimeState, startingRunId?: string
       let resolved: DaytonaSandbox;
       try {
         resolved = await state.provisioning.resolve(daytona);
-        if (!state.provisioning.isDesired(resolved)) {
-          resolved = await replaceSandboxRuntimeExclusive(state, daytona, resolved);
+        if (reason === "daytona_host_recovery" || !state.provisioning.isDesired(resolved)) {
+          resolved = await replaceSandboxRuntimeExclusive(state, daytona, resolved, reason);
         }
       } catch (error) {
         throw toUpstreamError(
@@ -412,6 +432,7 @@ async function replaceSandboxRuntimeExclusive(
   state: RuntimeState,
   daytona: DaytonaClient,
   current: DaytonaSandbox,
+  reason: RuntimeReplacementReason,
 ): Promise<DaytonaSandbox> {
   state.provisioning.assertRuntimeReplacementSafe(current);
   await prepareForSandboxReplacement(state);
@@ -421,6 +442,7 @@ async function replaceSandboxRuntimeExclusive(
     throw sandboxRuntimeUpdatePending(state.env.DAYTONA_SANDBOX_SNAPSHOT);
   }
   createLogger().info("sandbox_runtime_replaced", {
+    reason,
     sandboxId: state.identity.sandboxName(),
     snapshot: state.env.DAYTONA_SANDBOX_SNAPSHOT,
   });

packages/agent-core/src/tools/code/daytona-client.ts

@@ -34,6 +34,8 @@ const DAYTONA_FILE_LIST_MAX_ITEMS = 1_000;
 const DAYTONA_SANDBOX_PAGE_MAX_ITEMS = 100;
 const DAYTONA_SESSION_COMMAND_MAX_ITEMS = 1_000;
 const DAYTONA_VOLUME_NAME_MAX_CHARACTERS = 100;
+const DAYTONA_HOST_RECOVERY_START_MESSAGE =
+  "sandbox start is temporarily unavailable while the sandbox's host recovers";
 
 interface DaytonaClientConfig {
   apiKey: string;
@@ -66,6 +68,22 @@ export class DaytonaApiError extends Error {
   }
 }
 
+/** Identifies Daytona's host-local start rejection so callers can fail over safely. */
+export function isDaytonaHostRecoveryStartError(error: unknown): boolean {
+  let current = error;
+  for (let depth = 0; depth < 3; depth += 1) {
+    if (
+      current instanceof DaytonaApiError &&
+      current.status === 503 &&
+      current.message.toLowerCase().includes(DAYTONA_HOST_RECOVERY_START_MESSAGE)
+    ) {
+      return true;
+    }
+    current = current instanceof Error ? current.cause : undefined;
+  }
+  return false;
+}
+
 // ---------------------------------------------------------------------------
 // Response schemas project provider payloads down to fields used by the runtime.
 // ---------------------------------------------------------------------------

packages/agent-core/src/tools/code/index.ts

@@ -5,7 +5,11 @@ export type {
   DaytonaVolume,
   SandboxDestroyResult,
 } from "./daytona-client";
-export { DaytonaApiError, DaytonaClient } from "./daytona-client";
+export {
+  DaytonaApiError,
+  DaytonaClient,
+  isDaytonaHostRecoveryStartError,
+} from "./daytona-client";
 
 export {
   DeleteFileInputSchema,