fix(research): bound nested model synthesis (#167)

## Why

Production deep-research runs repeatedly failed in the per-query
synthesis step after Anthropic returned HTTP 524. Each pass sent a large
evidence pack with the default 4,096-token output budget, and the
synthesis stage allowed 8,192 tokens.

## What changed

- Bound Exa and Firecrawl evidence included in each model pass.
- Use stage-appropriate 2,048-token pass and 4,096-token final synthesis
budgets.
- Bound every nested research generation to 75 seconds.
- Retry one transient timeout, rate-limit, or 5xx failure in memory
while preserving parent cancellation.
- Emit a structured warning when that retry is used.
- Document the request-scoped retry and non-persistence boundary.

## Architecture and data

No schema, migration, environment, vendor, or deployment-topology
changes. BYOK credentials remain request-scoped and no Mastra workflow
snapshot contains secret-bearing state.

## Verification

- `pnpm lint`
- `pnpm typecheck`
- `pnpm turbo build --force`
- `pnpm deadcode`
- `pnpm architecture:check`
- `pnpm turbo skills:build`
- Production Worker trace reproduced the original HTTP 524 twice before
the fix.

The repository pins Node 24.18.0; local verification ran on Node 26.4.0
and emitted only the existing engine warning.

Author: iamjr15

packages/agent-core/README.md

@@ -67,10 +67,12 @@ the Mastra workflow run, forward each workflow step signal through every nested
 `agent.generate`, and remove abort listeners before deleting the ephemeral run.
 Each concurrent research pass gets an isolated evidence collector populated only
 from one bounded Exa discovery call and an optional Firecrawl extraction of its
-primary result. A single tool-free model pass structures each provider evidence
-pack, and workflow steps do not retry failed model output inside the same Worker
-invocation. Claim citations and the final synthesis are schema-validated against
-that evidence; prose URL scraping is not an accepted provenance boundary.
+primary result. A single tool-free model pass structures each byte-bounded provider
+evidence pack. Nested model calls use stage-appropriate output bounds, an operational
+timeout, and one in-memory retry for transient provider failures; request cancellation
+always wins and no secret-bearing state is snapshotted. Claim citations and the final
+synthesis are schema-validated against that evidence; prose URL scraping is not an
+accepted provenance boundary.
 Successful top-level deep-research and fan-out tools render the validated report's
 canonical GitHub-flavored Markdown directly into a PDF artifact. The chat response
 and PDF therefore preserve the same headings, prose, lists, tables, links, citations,

packages/agent-core/src/mastra/workflows/deep-research-workflow.ts

@@ -1,4 +1,4 @@
-import { APIError } from "@cheatcode/observability";
+import { APIError, createLogger } from "@cheatcode/observability";
 import { createStep, createWorkflow } from "@mastra/core/workflows";
 import { z } from "zod/v4";
 import {
@@ -28,9 +28,14 @@ import {
 } from "./research-schemas";
 
 const RESEARCH_QUERY_CONCURRENCY = 3;
-const RESEARCH_RESULTS_PER_QUERY = 6;
-const RESEARCH_RESULT_TEXT_CHARACTERS = 2_500;
-const RESEARCH_SCRAPE_CHARACTERS = 12_000;
+const RESEARCH_RESULTS_PER_QUERY = 5;
+const RESEARCH_RESULT_TEXT_CHARACTERS = 1_600;
+const RESEARCH_EVIDENCE_CHARACTERS_PER_SOURCE = 3_000;
+const RESEARCH_SCRAPE_CHARACTERS = 6_000;
+const RESEARCH_PASS_MAX_OUTPUT_TOKENS = 2_048;
+const RESEARCH_SYNTHESIS_MAX_OUTPUT_TOKENS = 4_096;
+const RESEARCH_MODEL_TIMEOUT_MS = 75_000;
+const RESEARCH_MODEL_ATTEMPTS = 2;
 const RESEARCH_PROVIDER_OPTIONS = {
   anthropic: { structuredOutputMode: "outputFormat" as const },
 };
@@ -143,13 +148,16 @@ function createQueryStep(id: string, config: ResearchWorkflowPrompts) {
         research.requestContext,
         abortSignal,
       );
-      const response = await agent.generate(researchPassPrompt(config, inputData.query, evidence), {
-        activeTools: [],
-        abortSignal,
-        providerOptions: RESEARCH_PROVIDER_OPTIONS,
-        requestContext: research.requestContext,
-        structuredOutput: { schema: ResearchPassDraftSchema },
-      });
+      const response = await generateResearchOutput(abortSignal, (generationSignal) =>
+        agent.generate(researchPassPrompt(config, inputData.query, evidence), {
+          activeTools: [],
+          abortSignal: generationSignal,
+          modelSettings: { maxOutputTokens: RESEARCH_PASS_MAX_OUTPUT_TOKENS },
+          providerOptions: RESEARCH_PROVIDER_OPTIONS,
+          requestContext: research.requestContext,
+          structuredOutput: { schema: ResearchPassDraftSchema },
+        }),
+      );
       return validateResearchPass(
         parseResearchPassDraft(response.object),
         inputData.query,
@@ -168,14 +176,16 @@ function createSynthesisStep(id: string, config: ResearchWorkflowPrompts) {
     execute: async ({ abortSignal, inputData, mastra, requestContext }) => {
       const agent = mastra.getAgent("general");
       const sources = mergeResearchSources(inputData);
-      const response = await agent.generate(researchSynthesisPrompt(config, inputData), {
-        activeTools: [],
-        abortSignal,
-        modelSettings: { maxOutputTokens: 8_192 },
-        providerOptions: RESEARCH_PROVIDER_OPTIONS,
-        requestContext,
-        structuredOutput: { schema: ResearchSynthesisDraftSchema },
-      });
+      const response = await generateResearchOutput(abortSignal, (generationSignal) =>
+        agent.generate(researchSynthesisPrompt(config, inputData), {
+          activeTools: [],
+          abortSignal: generationSignal,
+          modelSettings: { maxOutputTokens: RESEARCH_SYNTHESIS_MAX_OUTPUT_TOKENS },
+          providerOptions: RESEARCH_PROVIDER_OPTIONS,
+          requestContext,
+          structuredOutput: { schema: ResearchSynthesisDraftSchema },
+        }),
+      );
       const draft = parseResearchSynthesisDraft(response.object);
       return ResearchReportSchema.parse({
         claims: validateSynthesisClaims(draft.claims, sources),
@@ -239,7 +249,8 @@ function exaEvidenceSource(
 ): ResearchEvidenceSource {
   const content = [result.summary, ...result.highlights, result.text]
     .filter((value): value is string => Boolean(value))
-    .join("\n\n");
+    .join("\n\n")
+    .slice(0, RESEARCH_EVIDENCE_CHARACTERS_PER_SOURCE);
   return {
     content,
     provider: "exa",
@@ -335,6 +346,51 @@ function invalidStructuredResearchOutput(stage: string): APIError {
   );
 }
 
+async function generateResearchOutput<T>(
+  abortSignal: AbortSignal,
+  generate: (generationSignal: AbortSignal) => Promise<T>,
+): Promise<T> {
+  let lastError: unknown;
+  for (let attempt = 0; attempt < RESEARCH_MODEL_ATTEMPTS; attempt += 1) {
+    abortSignal.throwIfAborted();
+    const generationSignal = AbortSignal.any([
+      abortSignal,
+      AbortSignal.timeout(RESEARCH_MODEL_TIMEOUT_MS),
+    ]);
+    try {
+      return await generate(generationSignal);
+    } catch (error) {
+      abortSignal.throwIfAborted();
+      lastError = error;
+      if (!isRetriableModelError(error) || attempt === RESEARCH_MODEL_ATTEMPTS - 1) {
+        throw error;
+      }
+      createLogger().warn("research_model_generation_retrying", {
+        attempt: attempt + 1,
+        error,
+      });
+    }
+  }
+  throw lastError;
+}
+
+function isRetriableModelError(error: unknown): boolean {
+  if (!(error instanceof Error)) {
+    return false;
+  }
+  if (error.name === "TimeoutError") {
+    return true;
+  }
+  const record = error as Error & { isRetryable?: unknown; statusCode?: unknown };
+  if (record.isRetryable === true) {
+    return true;
+  }
+  return (
+    typeof record.statusCode === "number" &&
+    (record.statusCode === 408 || record.statusCode === 429 || record.statusCode >= 500)
+  );
+}
+
 function providerStatusIsSuccessful(statusCode: number | undefined): boolean {
   return statusCode === undefined || (statusCode >= 200 && statusCode < 400);
 }