fix(sandbox): rotate stale runtimes safely (#86)

## Summary

- replace a user's stale Daytona runtime container on the first
workspace operation after a snapshot or target promotion
- preserve the canonical persistent-volume subpath, project files,
uploaded files, and user-installed skills
- fence concurrent operations and active run leases during replacement,
while returning retriable maintenance responses
- clear only ephemeral process reservations and runtime projections
after the new container starts
- require canonical ownership, matching snapshot attestation, and the
exact workspace mount before automatic deletion; ambiguous contracts
still fail closed
- document the production promotion and recovery contract

## Why

Promoting the Node 24 snapshot made every existing sandbox permanently
return `unavailable_maintenance`: lookup detected the old snapshot, but
no lifecycle path ever replaced it. This change makes immutable snapshot
promotion operational without moving or deleting user workspace data.

The replacement decision is intentionally narrow. Snapshot or target
drift is recoverable only when the existing resource is unambiguously
the user's canonical sandbox on the configured persistent volume.
Identity, label, mount, and duplicate-resource ambiguity remain
operator-visible failures.

## Verification

- `pnpm turbo lint typecheck build` under Node 24.18.0
- agent-worker lint, typecheck, and Wrangler dry-run build under Node
24.18.0
- dependency-cruiser architecture check for `apps/agent-worker`
- Knip dead-code check for `@cheatcode/agent-worker`
- `git diff --check`
- exactly one SQL migration exists:
`packages/db/drizzle/0000_current_schema.sql`
- production migration dry-run reports that migration applied and
verifies the schema contract

Production upload/persistence acceptance QA will run directly through
`agent-browser` after the exact merged revision is deployed.

Author: iamjr15

apps/agent-worker/README.md

@@ -138,15 +138,17 @@ Each user has one durable Daytona sandbox. Projects are lexically confined to th
 folders under `/workspace`, and run leases keep the sandbox active while the agent is
 working. Project folders share the sandbox's Unix identity, so this prevents accidental
 cross-project access but is not an operating-system security boundary within one user.
-Sandbox lookup validates canonical ownership labels before trusting a cached ID; the
-physical Daytona name is deliberately not identity because a promoted replacement has a
-release-scoped name. A missing/stale Durable Object cache therefore recovers the one
-canonical sandbox by labels, while duplicate live canonical matches fail closed. New
-sandboxes pin the configured immutable snapshot and mount the environment's shared Daytona
-volume at `/workspace` with the user sandbox name as its isolated subpath. Canonical and
-candidate checks require the provider's actual mount tuple as well as the matching labels; labels
-alone cannot attest durable storage. A noncurrent
-sandbox is maintenance-only and cannot serve product work.
+Sandbox lookup validates canonical ownership labels before trusting a cached Daytona
+resource ID. A missing/stale Durable Object cache therefore recovers the one canonical
+sandbox by labels, while duplicate live canonical matches fail closed. New sandboxes pin
+the configured immutable snapshot and mount the environment's shared Daytona volume at
+`/workspace` with the user sandbox name as its isolated subpath. Canonical and candidate
+checks require the provider's actual mount tuple as well as the matching labels; labels
+alone cannot attest durable storage. When the configured snapshot or target changes, the
+first operation after active work drains replaces only the stale container and remounts
+that same volume subpath. New operations are fenced during replacement, stale process
+projections are cleared, and project files plus user-installed skills remain durable.
+Identity, snapshot-label, or storage-mount ambiguity still fails closed.
 
 Preview URLs carry a 60-second `handoff` capability minted by `@cheatcode/auth`.
 The preview-proxy Worker exchanges it for a distinct host-only, HttpOnly
@@ -229,10 +231,12 @@ application secret is required.
 Every ProjectSandbox uses the one configured immutable Daytona snapshot and the
 one configured shared workspace volume. Existing sandbox identity is accepted
 only when its owner, canonical labels, snapshot, volume, and mount contract all
-match. Mismatches fail closed instead of running a hidden migration. New
+match. A stale snapshot or target is replaced automatically only when canonical
+ownership and the persistent mount are unambiguous and no other operation or run
+lease is active. All other contract mismatches fail closed. New and replacement
 sandboxes mount the user's isolated volume subpath directly at `/workspace`.
-Account deletion clears that subpath before deleting all exactly owned
-sandboxes, so persistent volume data does not outlive the account.
+Account deletion clears that subpath before deleting all exactly owned sandboxes,
+so persistent volume data does not outlive the account.
 
 Production deploys bind one immutable `CHEATCODE_RELEASE_SHA`. Health responses
 expose that identity so the deployment workflow can verify that service

apps/agent-worker/src/durable-objects/project-sandbox-daytona-identity.ts

@@ -20,19 +20,47 @@ export function canonicalSandboxLabels(input: {
 }
 
 export function isCanonicalSandbox(sandbox: DaytonaSandbox, sandboxName: string): boolean {
-  return sandbox.labels["app"] === APP_LABEL && sandbox.labels["sandboxId"] === sandboxName;
+  return (
+    sandbox.labels["app"] === APP_LABEL &&
+    sandbox.labels["role"] === "canonical" &&
+    sandbox.labels["sandboxId"] === sandboxName &&
+    sandbox.labels["sandboxOwner"] === sandboxName
+  );
+}
+
+export function isRuntimeReplaceableCanonicalSandbox(
+  sandbox: DaytonaSandbox,
+  input: { sandboxName: string; volumeName: string },
+): boolean {
+  const volumeId = sandbox.labels["workspaceVolumeId"];
+  return (
+    isCanonicalSandbox(sandbox, input.sandboxName) &&
+    sandbox.snapshot === sandbox.labels["snapshot"] &&
+    typeof volumeId === "string" &&
+    volumeId.length > 0 &&
+    sandbox.labels["workspaceVolumeName"] === input.volumeName &&
+    hasWorkspaceMount(sandbox, volumeId, input.sandboxName)
+  );
 }
 
 export function isDesiredCanonicalSandbox(
   sandbox: DaytonaSandbox,
-  input: { sandboxName: string; snapshot: string; volumeId?: string; volumeName: string },
+  input: {
+    sandboxName: string;
+    snapshot: string;
+    target: string;
+    volumeId?: string;
+    volumeName: string;
+  },
 ): boolean {
   const volumeId = input.volumeId ?? sandbox.labels["workspaceVolumeId"];
   return (
     isCanonicalSandbox(sandbox, input.sandboxName) &&
     sandbox.labels["role"] === "canonical" &&
     sandbox.snapshot === input.snapshot &&
     sandbox.labels["snapshot"] === input.snapshot &&
+    sandbox.target === input.target &&
+    sandbox.user === "node" &&
     typeof volumeId === "string" &&
     volumeId.length > 0 &&
     sandbox.labels["workspaceVolumeId"] === volumeId &&

apps/agent-worker/src/durable-objects/project-sandbox-lifecycle.ts

@@ -34,6 +34,7 @@ import {
   setSandboxQuotaPeriod,
 } from "./project-sandbox-metering";
 import { assertProjectSandboxOwnerActive } from "./project-sandbox-owner-admission";
+import { PROC_PREFIX, PROCESS_PORT_ALLOC_KEY } from "./project-sandbox-process-support";
 import { ProjectSandboxProvisioning } from "./project-sandbox-provisioning";
 import type {
   ParsedProjectCleanupWorkspaceInput,
@@ -46,6 +47,9 @@ import {
 } from "./project-sandbox-workspace-state";
 
 const ClearWorkspaceEvidenceSchema = z.object({ cleared: z.literal(true) }).strict();
+const RUNTIME_MANIFEST_PATH = "/workspace/.cheatcode/runtime.json";
+const RUNTIME_RESET_PENDING_KEY = "sandbox_runtime_reset_pending";
+const SKILL_RUNTIME_DIRECTORY = "/workspace/.cheatcode/runtime";
 
 export abstract class ProjectSandboxLifecycle extends DurableObject<ProjectSandboxEnv> {
   private accountDeletionCompleted = false;
@@ -56,6 +60,7 @@ export abstract class ProjectSandboxLifecycle extends DurableObject<ProjectSandb
   private daytonaClient: DaytonaClient | undefined;
   private daytonaId: string | undefined;
   private sandboxMutationTail: Promise<void> = Promise.resolve();
+  private sandboxRuntimeUpdateInProgress = false;
   private startedVerifiedAtMs = 0;
   private readonly identityState: ProjectSandboxIdentityState;
   private readonly provisioning: ProjectSandboxProvisioning;
@@ -146,13 +151,15 @@ export abstract class ProjectSandboxLifecycle extends DurableObject<ProjectSandb
     operation: () => Promise<T>,
     isSharedMutation = false,
     shouldLeaseUnknownWorkspace = false,
+    allowRuntimeUpdate = false,
   ): Promise<T> {
     let release: (() => void) | undefined;
     try {
       release = this.acquireActiveOperation(
         workspaceScope,
         isSharedMutation,
         shouldLeaseUnknownWorkspace,
+        allowRuntimeUpdate,
       );
       return operation().finally(release);
     } catch (error) {
@@ -187,7 +194,7 @@ export abstract class ProjectSandboxLifecycle extends DurableObject<ProjectSandb
   protected withActiveSandboxCleanupSignal(operation: () => Promise<void>): Promise<void> {
     return this.accountDeletionInProgress || !this.identityState.hasRegisteredOwner()
       ? Promise.resolve()
-      : this.withActiveSandboxOperation(operation);
+      : this.withActiveOperation(null, operation, false, false, true);
   }
 
   private async initializeIdentityState(): Promise<void> {
@@ -245,7 +252,7 @@ export abstract class ProjectSandboxLifecycle extends DurableObject<ProjectSandb
     remaining.push({ runId, startedMs: Date.now() });
     await this.ctx.storage.put(RUN_LEASES_KEY, remaining);
     try {
-      const id = await this.ensureSandbox();
+      const id = await this.ensureSandbox(runId);
       await this.client()
         .setAutoStopInterval(id, 0)
         .catch(() => undefined);
@@ -441,10 +448,14 @@ export abstract class ProjectSandboxLifecycle extends DurableObject<ProjectSandb
   private acquireActiveSandboxOperation(
     allowUnregisteredOwner = false,
     allowWorkspaceCleanup = false,
+    allowRuntimeUpdate = false,
   ): () => void {
     if (this.accountDeletionInProgress) {
       throw accountSandboxDeletedError();
     }
+    if (this.sandboxRuntimeUpdateInProgress && !allowRuntimeUpdate) {
+      throw sandboxRuntimeUpdatePending(this.env.DAYTONA_SANDBOX_SNAPSHOT);
+    }
     if (!allowUnregisteredOwner && !this.identityState.hasRegisteredOwner()) {
       throw accountSandboxDeletedError();
     }
@@ -468,8 +479,9 @@ export abstract class ProjectSandboxLifecycle extends DurableObject<ProjectSandb
     workspaceScope: string | readonly string[] | null,
     isSharedMutation = false,
     shouldLeaseUnknownWorkspace = false,
+    allowRuntimeUpdate = false,
   ): () => void {
-    const releaseSandbox = this.acquireActiveSandboxOperation();
+    const releaseSandbox = this.acquireActiveSandboxOperation(false, false, allowRuntimeUpdate);
     let releaseWorkspace: (() => void) | undefined;
     try {
       const workspaceSlugs =
@@ -529,12 +541,12 @@ export abstract class ProjectSandboxLifecycle extends DurableObject<ProjectSandb
     this.startedVerifiedAtMs = Date.now();
   }
 
-  protected async ensureSandbox(): Promise<string> {
+  protected async ensureSandbox(startingRunId?: string): Promise<string> {
     return this.withSandboxMutation(async () => {
       if (this.daytonaId && Date.now() - this.startedVerifiedAtMs < STARTED_REVERIFY_MS) {
         return this.daytonaId;
       }
-      return this.resolveStartedSandbox();
+      return this.resolveStartedSandbox(startingRunId);
     });
   }
 
@@ -588,11 +600,14 @@ export abstract class ProjectSandboxLifecycle extends DurableObject<ProjectSandb
     }
   }
 
-  private async resolveStartedSandbox(): Promise<string> {
+  private async resolveStartedSandbox(startingRunId?: string): Promise<string> {
     const client = await this.ensureClient();
     let resolved: DaytonaSandbox;
     try {
       resolved = await this.provisioning.resolve(client);
+      if (!this.provisioning.isDesired(resolved)) {
+        resolved = await this.replaceSandboxRuntime(client, resolved, startingRunId);
+      }
     } catch (error) {
       throw this.toUpstreamError(error, "Daytona sandbox lookup failed.");
     }
@@ -603,10 +618,78 @@ export abstract class ProjectSandboxLifecycle extends DurableObject<ProjectSandb
         retriable: true,
       });
     }
+    await this.clearPersistedRuntimeProjection(client, resolved.id);
     this.startedVerifiedAtMs = Date.now();
     return resolved.id;
   }
 
+  private async replaceSandboxRuntime(
+    client: DaytonaClient,
+    current: DaytonaSandbox,
+    startingRunId?: string,
+  ): Promise<DaytonaSandbox> {
+    this.sandboxRuntimeUpdateInProgress = true;
+    try {
+      await this.assertSandboxReplacementAllowed(startingRunId);
+      this.provisioning.assertRuntimeReplacementSafe(current);
+      await this.prepareForSandboxReplacement();
+      await this.provisioning.deleteForReplacement(client, current);
+      const replacement = await this.provisioning.create(client);
+      if (!this.provisioning.isDesired(replacement)) {
+        throw sandboxRuntimeUpdatePending(this.env.DAYTONA_SANDBOX_SNAPSHOT);
+      }
+      createLogger().info("sandbox_runtime_replaced", {
+        sandboxId: this.sandboxName(),
+        snapshot: this.env.DAYTONA_SANDBOX_SNAPSHOT,
+      });
+      return replacement;
+    } finally {
+      this.sandboxRuntimeUpdateInProgress = false;
+    }
+  }
+
+  private async assertSandboxReplacementAllowed(startingRunId?: string): Promise<void> {
+    const leases = await this.runLeases();
+    const activeRunLeases = leases.filter(
+      (lease) => Date.now() - lease.startedMs < STALE_RUN_LEASE_MS,
+    );
+    if (activeRunLeases.length !== leases.length) {
+      await this.ctx.storage.put(RUN_LEASES_KEY, activeRunLeases);
+    }
+    const otherRunLeases = activeRunLeases.filter((lease) => lease.runId !== startingRunId);
+    if (this.activeOperationCount > 1 || otherRunLeases.length > 0) {
+      throw sandboxRuntimeUpdatePending(this.env.DAYTONA_SANDBOX_SNAPSHOT);
+    }
+  }
+
+  private async prepareForSandboxReplacement(): Promise<void> {
+    this.daytonaId = undefined;
+    this.startedVerifiedAtMs = 0;
+    await this.ctx.storage.delete(DAYTONA_ID_KEY);
+    await this.ctx.storage.put(RUNTIME_RESET_PENDING_KEY, true);
+    const processRecords = await this.ctx.storage.list({ prefix: PROC_PREFIX });
+    if (processRecords.size > 0) {
+      await this.ctx.storage.delete([...processRecords.keys()]);
+    }
+    await this.ctx.storage.delete(PROCESS_PORT_ALLOC_KEY);
+  }
+
+  private async clearPersistedRuntimeProjection(
+    client: DaytonaClient,
+    sandboxId: string,
+  ): Promise<void> {
+    if ((await this.ctx.storage.get(RUNTIME_RESET_PENDING_KEY)) !== true) {
+      return;
+    }
+    try {
+      await client.deleteFilePath(sandboxId, RUNTIME_MANIFEST_PATH, false);
+      await client.deleteFilePath(sandboxId, SKILL_RUNTIME_DIRECTORY, true);
+      await this.ctx.storage.delete(RUNTIME_RESET_PENDING_KEY);
+    } catch (error) {
+      throw this.toUpstreamError(error, "Daytona runtime reset failed.");
+    }
+  }
+
   protected async existingSandboxId(): Promise<string | null> {
     const client = await this.ensureClient();
     try {
@@ -708,3 +791,16 @@ export abstract class ProjectSandboxLifecycle extends DurableObject<ProjectSandb
     this.startedVerifiedAtMs = 0;
   }
 }
+
+function sandboxRuntimeUpdatePending(expectedSnapshot: string): APIError {
+  return new APIError(
+    503,
+    "unavailable_maintenance",
+    "This computer is updating to the current runtime.",
+    {
+      details: { expectedSnapshot },
+      hint: "Retry after the active operation finishes.",
+      retriable: true,
+    },
+  );
+}