fix(agent): prevent fourth-tool workflow failure (#181)

## Summary
- Fixes the deterministic fourth-tool crash caused by validating 16
rotated skill capabilities before trimming the persisted set to its
12-entry bound.
- Routes projectless imperatives such as “build a nice pomodoro app”
through the managed web app-builder unless the prompt explicitly selects
mobile or a non-web runtime.
- Adds run-scoped, redacted error telemetry and recovers known
structured error codes across workflow error wrappers.

## Root cause
Every checkpointed tool step minted four independently scoped runtime
capabilities. The fourth step combined twelve retained capabilities with
four new ones, validated the sixteen-entry intermediate array against a
twelve-entry schema, and failed before the requested shell command ran.
Workflow retries repeated the same deterministic validation failure.

## Architecture
Capability inputs are validated independently, then the
retained-plus-new set is trimmed to the durable storage bound before
final validation and persistence. App-builder inference remains a narrow
projectless fallback: explicit mobile signals win, explicit web signals
follow, and a generic app defaults to web unless a non-web runtime is
named.

Workflow terminal and pre-tool infrastructure errors now emit only
allowlisted categorical metadata. Prompts, commands, provider responses,
stack traces, and credentials remain excluded.

## Decisions made
| Decision | Choice | Reasoning |
|---|---|---|
| Capability rotation | Validate new values, trim combined values,
validate stored result | Preserves strict boundary validation without
rejecting the temporary overlap that the bound exists to prune. |
| Generic app routing | Default imperative `app` requests to managed web
app-builder | Matches normal user language while explicit mobile and
non-web signals remain authoritative. |
| Error transport | Add a canonical Zod error-code schema and safe
wrapper recovery | Prevents known error classifications from collapsing
solely because a platform boundary changes the Error prototype. |
| Failure telemetry | Emit run-scoped safe classifications at pre-tool
and terminal boundaries | Makes exhausted retries diagnosable without
retaining sensitive content. |

## Edge cases handled
- Explicit mobile prompts continue to use the mobile builder.
- CLI, backend, API, desktop, Electron, terminal, server, and library
prompts remain on the general path unless an explicit web signal is
present.
- Invalid new capabilities cannot be hidden by trimming because the
incoming set is validated first.
- The unrelated `dogfood-output/` working directory is not included.

## Verification
- [x] `pnpm lint`
- [x] `pnpm typecheck`
- [x] `pnpm turbo build --force`
- [x] `pnpm deadcode`
- [x] `pnpm architecture:check`
- [x] `pnpm turbo skills:build`
- [ ] Deploy the merged SHA to production
- [ ] Exercise “build a nice pomodoro app” on production through
scaffold, four-plus tool calls, dev-server start, and live preview

Author: iamjr15

apps/agent-worker/README.md

@@ -122,10 +122,12 @@ streams. The Workflow controller owns admission, callback identity, and cancella
 retains only durable run identity, status, transcript, cancellation, and dependency wiring.
 
 An explicit app-builder mode remains authoritative. In a projectless chat, a narrowly
-matched imperative such as “build a website” or “create a mobile app” also enters the matching
-app-builder path before model execution. That high-confidence fallback materializes the project,
-scaffolds its canonical workspace, and registers the managed preview even when the selected model
-would otherwise attempt generic shell work and finish without a Computer target.
+matched imperative such as “build a website,” “create a pomodoro app,” or “create a mobile app”
+also enters the matching app-builder path before model execution. A generic app defaults to the
+web path unless the request carries an explicit mobile or non-web runtime signal. That
+high-confidence fallback materializes the project, scaffolds its canonical workspace, and
+registers the managed preview even when the selected model would otherwise attempt generic shell
+work and finish without a Computer target.
 The starter page is an internal server-readiness target, not user-facing generated content. A
 fresh template run emits the typed `app-preview-status` transition from `building` to `ready` only
 after model execution (and the mobile preview restart, when applicable), so the web client can keep

apps/agent-worker/src/durable-objects/agent-run-errors.ts

@@ -1,5 +1,5 @@
-import { APIError } from "@cheatcode/observability";
-import type { ErrorCode } from "@cheatcode/types";
+import { APIError, safeErrorTelemetry } from "@cheatcode/observability";
+import { type ErrorCode, ErrorCodeSchema } from "@cheatcode/types";
 
 export interface AgentRunStreamError {
   code: ErrorCode;
@@ -16,9 +16,41 @@ export function toAgentRunStreamError(error: unknown): AgentRunStreamError {
     };
   }
 
+  const telemetry = safeErrorTelemetry(error);
+  const code = errorCodeFromTelemetry(telemetry.sourceErrorCode, telemetry.causeCode);
+  if (code) {
+    return {
+      code,
+      message: workflowFailureMessage(code),
+      retriable: telemetry.retriable ?? telemetry.causeRetriable ?? true,
+    };
+  }
+
   return {
     code: "tool_execution_failed",
     message: "Agent run failed unexpectedly",
     retriable: true,
   };
 }
+
+function errorCodeFromTelemetry(...candidates: Array<string | undefined>): ErrorCode | undefined {
+  for (const candidate of candidates) {
+    const parsed = ErrorCodeSchema.safeParse(candidate);
+    if (parsed.success) return parsed.data;
+  }
+  return undefined;
+}
+
+function workflowFailureMessage(code: ErrorCode): string {
+  if (
+    code === "sandbox_start_failed" ||
+    code === "upstream_sandbox_failed" ||
+    code === "upstream_sandbox_timeout"
+  ) {
+    return "The computer could not complete this operation";
+  }
+  if (code === "internal_service_error" || code === "service_maintenance_unavailable") {
+    return "The agent service could not complete this run";
+  }
+  return "Agent run failed before it could finish";
+}

apps/agent-worker/src/durable-objects/agent-run-path.ts

@@ -97,6 +97,9 @@ const IMPERATIVE_BUILD_PATTERN =
 const MOBILE_APP_PATTERN = /\b(?:mobile app|expo|react native|ios app|android app|iphone app)\b/iu;
 const WEB_APP_PATTERN =
   /\b(?:web ?app|website|web ?site|landing ?page|home ?page|web ?page|dashboard|next\.?js|frontend|front-end|saas)\b/iu;
+const GENERIC_APP_PATTERN = /\b(?:app|application)\b/iu;
+const NON_WEB_APP_PATTERN =
+  /\b(?:api|backend|cli|command[ -]line|desktop|electron|library|macos|server|terminal|windows)\b/iu;
 
 function appBuilderModeForRun(input: StartRunInput): "app-builder" | "app-builder-mobile" | null {
   if (isAppBuilderMode(input.projectMode)) {
@@ -108,5 +111,10 @@ function appBuilderModeForRun(input: StartRunInput): "app-builder" | "app-builde
   if (MOBILE_APP_PATTERN.test(input.messageText)) {
     return "app-builder-mobile";
   }
-  return WEB_APP_PATTERN.test(input.messageText) ? "app-builder" : null;
+  if (WEB_APP_PATTERN.test(input.messageText)) {
+    return "app-builder";
+  }
+  return GENERIC_APP_PATTERN.test(input.messageText) && !NON_WEB_APP_PATTERN.test(input.messageText)
+    ? "app-builder"
+    : null;
 }

apps/agent-worker/src/durable-objects/agent-run-workflow-runtime.ts

@@ -3,7 +3,13 @@ import {
   type GeneralAgentToolCall,
   generateGeneralAgentStep,
 } from "@cheatcode/agent-core";
-import { createLogger, emitUserEvent, readBoundedResponseJson } from "@cheatcode/observability";
+import {
+  createLogger,
+  emitErrorEvent,
+  emitUserEvent,
+  readBoundedResponseJson,
+  safeErrorTelemetry,
+} from "@cheatcode/observability";
 import type { ArtifactRuntime, WorkspaceResolver } from "@cheatcode/sandbox-contracts";
 import {
   FALLBACK_MODEL_ID,
@@ -21,6 +27,7 @@ import { storeAgentArtifact } from "./agent-run-artifacts";
 import { loadThreadModelContext } from "./agent-run-conversation";
 import { restoreReferencedDeliverables } from "./agent-run-deliverables";
 import type { AgentRunEnv } from "./agent-run-env";
+import { toAgentRunStreamError } from "./agent-run-errors";
 import {
   createAgentRequestContext,
   type MastraContextOptions,
@@ -211,24 +218,56 @@ export async function executeWorkflowToolStep(
     userId: input.userId,
   });
   const sandbox = sandboxFor(env, input);
-  await sandbox.renewRun(input.runId);
-  await stub.waitForBrowserTakeover(callback);
-  return guardSkillRuntimeCapabilities({
-    env,
-    logger,
-    operation: () =>
-      executePreparedWorkflowTool({
-        callback,
-        env,
-        input,
-        logger,
-        sandbox,
-        selectedLogicalModelId: state.selectedLogicalModelId,
-        stub,
-        toolCall,
-      }),
-    run: input,
-    sandbox,
+  try {
+    await sandbox.renewRun(input.runId);
+    await stub.waitForBrowserTakeover(callback);
+    return await guardSkillRuntimeCapabilities({
+      env,
+      logger,
+      operation: () =>
+        executePreparedWorkflowTool({
+          callback,
+          env,
+          input,
+          logger,
+          sandbox,
+          selectedLogicalModelId: state.selectedLogicalModelId,
+          stub,
+          toolCall,
+        }),
+      run: input,
+      sandbox,
+    });
+  } catch (error) {
+    recordToolStepInfrastructureFailure(env, input, toolCall, error, logger);
+    throw error;
+  }
+}
+
+function recordToolStepInfrastructureFailure(
+  env: AgentRunEnv,
+  input: StartRunInput,
+  toolCall: GeneralAgentToolCall,
+  error: unknown,
+  logger: ReturnType<typeof createLogger>,
+): void {
+  const failure = toAgentRunStreamError(error);
+  const telemetry = safeErrorTelemetry(error);
+  emitErrorEvent(env, {
+    errorCategory: "agent_run_tool_step",
+    errorCode: failure.code,
+    route: "agent-run-workflow/tool-step",
+    runId: input.runId,
+    userId: input.userId,
+    workerName: "agent-worker",
+    ...(env.CHEATCODE_RELEASE_SHA ? { versionTag: env.CHEATCODE_RELEASE_SHA } : {}),
+    ...telemetry,
+  });
+  logger.error("agent_run_tool_step_failed", {
+    failureCode: failure.code,
+    toolCallId: toolCall.toolCallId,
+    toolName: toolCall.toolName,
+    ...telemetry,
   });
 }
 

apps/agent-worker/src/durable-objects/agent-run-workflow.ts

@@ -1,9 +1,12 @@
 import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep } from "cloudflare:workers";
 import { NonRetryableError } from "cloudflare:workflows";
 import {
+  createLogger,
+  emitErrorEvent,
   emitUserEvent,
   readBoundedResponseJson,
   readBoundedResponseText,
+  safeErrorTelemetry,
 } from "@cheatcode/observability";
 import type { ModelMessage, ToolResultPart, UIMessageChunk } from "ai";
 import { z } from "zod";
@@ -92,6 +95,7 @@ export class AgentRunWorkflow extends WorkflowEntrypoint<
       return { status: "completed" };
     } catch (error) {
       const failure = toAgentRunStreamError(error);
+      recordWorkflowFailure(this.env, payload, event.instanceId, failure, error);
       await step.do("fail AgentRun", FAILURE_STEP, () =>
         failAgentRun(this.env, event.instanceId, payload, failure),
       );
@@ -104,6 +108,35 @@ export class AgentRunWorkflow extends WorkflowEntrypoint<
   }
 }
 
+function recordWorkflowFailure(
+  env: AgentRunWorkflowEnv,
+  payload: AgentRunWorkflowPayload,
+  workflowInstanceId: string,
+  failure: ReturnType<typeof toAgentRunStreamError>,
+  error: unknown,
+): void {
+  const telemetry = safeErrorTelemetry(error);
+  emitErrorEvent(env, {
+    errorCategory: "agent_run_workflow",
+    errorCode: failure.code,
+    route: "agent-run-workflow",
+    runId: payload.input.runId,
+    userId: payload.input.userId,
+    workerName: "agent-worker",
+    ...(env.CHEATCODE_RELEASE_SHA ? { versionTag: env.CHEATCODE_RELEASE_SHA } : {}),
+    ...telemetry,
+  });
+  createLogger({
+    runId: payload.input.runId,
+    threadId: payload.input.threadId,
+    userId: payload.input.userId,
+  }).error("agent_run_workflow_failed", {
+    failureCode: failure.code,
+    workflowInstanceId,
+    ...telemetry,
+  });
+}
+
 export async function admitAgentRunWorkflow(
   env: AgentRunWorkflowBindings,
   payload: AgentRunWorkflowPayload,

packages/db/src/skill-runtime-capabilities.ts

@@ -54,10 +54,10 @@ export async function rotateSkillRuntimeCapabilities(
     const retained = StoredSkillRuntimeCapabilitiesSchema.parse(run.capabilities).filter(
       (capability) => capability.expiresAt > input.now,
     );
-    const capabilities = StoredSkillRuntimeCapabilitiesSchema.parse([
-      ...retained,
-      ...input.capabilities,
-    ]).slice(-MAX_STORED_CAPABILITIES_PER_RUN);
+    const next = StoredSkillRuntimeCapabilitiesSchema.parse(input.capabilities);
+    const capabilities = StoredSkillRuntimeCapabilitiesSchema.parse(
+      [...retained, ...next].slice(-MAX_STORED_CAPABILITIES_PER_RUN),
+    );
     const [updated] = await tx
       .update(agentRuns)
       .set({ skillRuntimeCapabilities: capabilities })

packages/observability/README.md

@@ -41,6 +41,9 @@ Error Analytics Engine rows intentionally contain only categorical metadata.
 Raw error messages and stack traces are never written to Analytics Engine, and
 the structured logger suppresses error-message, stack, SQL, parameter, body,
 prompt, content, and command-output fields at its sink.
+Agent Workflow terminal failures and pre-tool infrastructure failures emit run-scoped rows through
+this same safe projection, so an exhausted retry retains source and cause classifications without
+persisting prompts, commands, provider responses, or credentials.
 
 ## Code Checks
 

packages/types/src/errors.ts

@@ -1,58 +1,61 @@
 import { z } from "zod";
 
-export type ErrorCode =
-  | "auth_token_missing"
-  | "auth_token_invalid"
-  | "auth_token_expired"
-  | "payment_method_required"
-  | "payment_method_failed"
-  | "subscription_past_due"
-  | "permission_access_denied"
-  | "permission_plan_required"
-  | "resource_user_not_found"
-  | "resource_project_not_found"
-  | "resource_thread_not_found"
-  | "resource_run_not_found"
-  | "resource_output_not_found"
-  | "resource_tool_not_found"
-  | "resource_skill_not_found"
-  | "request_body_invalid"
-  | "request_query_param_invalid"
-  | "request_path_param_invalid"
-  | "validation_model_unavailable"
-  | "validation_tool_not_registered"
-  | "idempotency_key_reused"
-  | "validation_byok_required"
-  | "conflict_request_in_flight"
-  | "conflict_run_already_active"
-  | "conflict_state_invalid"
-  | "rate_limit_exceeded"
-  | "quota_sandbox_hours_exhausted"
-  | "quota_composio_calls_exhausted"
-  | "byok_key_missing"
-  | "byok_key_invalid"
-  | "byok_key_quota_exhausted"
-  | "sandbox_disk_full"
-  | "sandbox_cpu_exhausted"
-  | "sandbox_start_failed"
-  | "sandbox_command_failed"
-  | "sandbox_process_limit_reached"
-  | "tool_validation_failed"
-  | "tool_execution_failed"
-  | "tool_execution_timeout"
-  | "upstream_llm_overloaded"
-  | "upstream_llm_failed"
-  | "upstream_llm_timeout"
-  | "upstream_sandbox_failed"
-  | "upstream_sandbox_timeout"
-  | "upstream_provider_outage"
-  | "repo_import_failed"
-  | "internal_service_error"
-  | "service_maintenance_unavailable";
+export const ErrorCodeSchema = z.enum([
+  "auth_token_missing",
+  "auth_token_invalid",
+  "auth_token_expired",
+  "payment_method_required",
+  "payment_method_failed",
+  "subscription_past_due",
+  "permission_access_denied",
+  "permission_plan_required",
+  "resource_user_not_found",
+  "resource_project_not_found",
+  "resource_thread_not_found",
+  "resource_run_not_found",
+  "resource_output_not_found",
+  "resource_tool_not_found",
+  "resource_skill_not_found",
+  "request_body_invalid",
+  "request_query_param_invalid",
+  "request_path_param_invalid",
+  "validation_model_unavailable",
+  "validation_tool_not_registered",
+  "idempotency_key_reused",
+  "validation_byok_required",
+  "conflict_request_in_flight",
+  "conflict_run_already_active",
+  "conflict_state_invalid",
+  "rate_limit_exceeded",
+  "quota_sandbox_hours_exhausted",
+  "quota_composio_calls_exhausted",
+  "byok_key_missing",
+  "byok_key_invalid",
+  "byok_key_quota_exhausted",
+  "sandbox_disk_full",
+  "sandbox_cpu_exhausted",
+  "sandbox_start_failed",
+  "sandbox_command_failed",
+  "sandbox_process_limit_reached",
+  "tool_validation_failed",
+  "tool_execution_failed",
+  "tool_execution_timeout",
+  "upstream_llm_overloaded",
+  "upstream_llm_failed",
+  "upstream_llm_timeout",
+  "upstream_sandbox_failed",
+  "upstream_sandbox_timeout",
+  "upstream_provider_outage",
+  "repo_import_failed",
+  "internal_service_error",
+  "service_maintenance_unavailable",
+]);
+
+export type ErrorCode = z.infer<typeof ErrorCodeSchema>;
 
 export const ErrorResponseSchema = z.strictObject({
   error: z.strictObject({
-    code: z.string(),
+    code: ErrorCodeSchema,
     message: z.string(),
     hint: z.string().optional(),
     retriable: z.boolean(),

packages/types/src/index.ts

@@ -31,7 +31,7 @@ export {
 } from "./billing";
 export type { AgentCapabilityName, ToolCapabilityName } from "./capabilities";
 export type { ErrorCode } from "./errors";
-export { ErrorResponseSchema } from "./errors";
+export { ErrorCodeSchema, ErrorResponseSchema } from "./errors";
 export type { AgentRunId, ProjectId, ThreadId, UserId } from "./ids";
 export { toAgentRunId, toProjectId, toThreadId, toUserId } from "./ids";
 export type { IntegrationName } from "./integrations";