fix: bound docs manifest fan-out to stop long hangs on redirect-miss docs requests

src/utils/docs.functions.ts

@@ -3,21 +3,14 @@ import { createServerFn } from '@tanstack/react-start'
 import { setResponseHeader } from '@tanstack/react-start/server'
 import removeMarkdown from 'remove-markdown'
 import * as v from 'valibot'
-import {
-  extractFrontMatter,
-  fetchApiContents,
-  fetchRepoFile,
-  isRecoverableGitHubContentError,
-  shouldUseLocalDocsFiles,
-} from '~/utils/documents.server'
 import { extractFrameworksFromMarkdown } from './markdown/filterFrameworkContent'
-import { getCachedDocsArtifact } from './github-content-cache.server'
 import { buildRedirectManifest, type RedirectManifestEntry } from './redirects'
 import { isValidRepoPath, MAX_REPO_PATH_LENGTH } from './repo-path'
 import { removeLeadingSlash } from './utils'
 import type { DocsRedirectManifest } from './docs-redirects'
+import type { GitHubFileNode } from './documents.server'
 
-type DocsTreeNode = {
+export type DocsTreeNode = {
   path: string
   children?: Array<DocsTreeNode>
 }
@@ -92,10 +85,45 @@ const docsRedirectInput = v.object({
   docsPaths: v.array(v.pipe(v.string(), v.maxLength(512))),
 })
 
+// Matches RAW_FETCH_CONCURRENCY in github-example.server.ts.
+const DOCS_MANIFEST_FETCH_CONCURRENCY = 6
+
+export async function mapWithConcurrency<T, TResult>(
+  values: Array<T>,
+  concurrency: number,
+  fn: (value: T) => Promise<TResult>,
+) {
+  const results = new Array<TResult>(values.length)
+  let index = 0
+
+  const workers = Array.from(
+    { length: Math.min(concurrency, values.length) },
+    async () => {
+      while (index < values.length) {
+        const currentIndex = index
+        index += 1
+        results[currentIndex] = await fn(values[currentIndex])
+      }
+    },
+  )
+
+  await Promise.all(workers)
+
+  return results
+}
+
 const temporarilyUnavailableMarkdown = `# Content temporarily unavailable
 
 We are having trouble fetching this document from GitHub right now. Please try again in a minute.`
 
+async function loadDocumentsServerModule() {
+  return import('./documents.server')
+}
+
+async function loadGitHubContentCacheServerModule() {
+  return import('./github-content-cache.server')
+}
+
 function buildUnavailableFile(filePath: string) {
   if (filePath.toLowerCase().endsWith('.md')) {
     return temporarilyUnavailableMarkdown
@@ -109,6 +137,9 @@ async function readRepoFileOrFallback(
   branch: string,
   filePath: string,
 ) {
+  const { fetchRepoFile, isRecoverableGitHubContentError } =
+    await loadDocumentsServerModule()
+
   try {
     return await fetchRepoFile(repo, branch, filePath)
   } catch (error) {
@@ -146,6 +177,64 @@ function isDocsManifest(value: unknown): value is DocsManifest {
   )
 }
 
+// Extracted so tests can inject a fake fetchFile without hitting real
+// GitHub network/cache.
+export async function collectRedirectEntriesForFile(
+  node: DocsTreeNode,
+  opts: {
+    docsRoot: string
+    fetchFile: (filePath: string) => Promise<string | null>
+    onCanonicalPath: (canonicalPath: string) => void
+  },
+): Promise<Array<RedirectManifestEntry>> {
+  const { extractFrontMatter, isRecoverableGitHubContentError } =
+    await loadDocumentsServerModule()
+  const canonicalPath = getCanonicalDocsPath(node.path, opts.docsRoot)
+
+  if (canonicalPath === null) {
+    return []
+  }
+
+  opts.onCanonicalPath(canonicalPath)
+
+  let file: string | null
+  try {
+    file = await opts.fetchFile(node.path)
+  } catch (error) {
+    if (!isRecoverableGitHubContentError(error)) {
+      throw error
+    }
+
+    return []
+  }
+
+  if (!file) {
+    return []
+  }
+
+  const frontMatter = extractFrontMatter(file)
+  const entries: Array<RedirectManifestEntry> = []
+
+  for (const redirectFrom of frontMatter.data.redirectFrom ?? []) {
+    const normalizedRedirect = normalizeDocsRedirectPath(
+      redirectFrom,
+      opts.docsRoot,
+    )
+
+    if (!normalizedRedirect || normalizedRedirect === canonicalPath) {
+      continue
+    }
+
+    entries.push({
+      from: normalizedRedirect,
+      to: canonicalPath,
+      source: node.path,
+    })
+  }
+
+  return entries
+}
+
 async function buildDocsManifest({
   repo,
   branch,
@@ -155,6 +244,7 @@ async function buildDocsManifest({
   branch: string
   docsRoot: string
 }): Promise<DocsManifest> {
+  const { fetchApiContents, fetchRepoFile } = await loadDocumentsServerModule()
   const nodes = await fetchApiContents(repo, branch, docsRoot)
 
   if (!nodes) {
@@ -165,46 +255,23 @@ async function buildDocsManifest({
     node.path.endsWith('.md'),
   )
   const paths = new Set<string>()
-  const redirects: Array<RedirectManifestEntry> = []
 
-  for (const node of markdownFiles) {
-    const canonicalPath = getCanonicalDocsPath(node.path, docsRoot)
-
-    if (canonicalPath === null) {
-      continue
-    }
-
-    paths.add(canonicalPath)
-
-    const file = await fetchRepoFile(repo, branch, node.path)
-
-    if (!file) {
-      continue
-    }
-
-    const frontMatter = extractFrontMatter(file)
-
-    for (const redirectFrom of frontMatter.data.redirectFrom ?? []) {
-      const normalizedRedirect = normalizeDocsRedirectPath(
-        redirectFrom,
+  // A recoverable error on one file must not fail the whole manifest build
+  // (see collectRedirectEntriesForFile).
+  const redirectsByFile = await mapWithConcurrency(
+    markdownFiles,
+    DOCS_MANIFEST_FETCH_CONCURRENCY,
+    (node) =>
+      collectRedirectEntriesForFile(node, {
         docsRoot,
-      )
-
-      if (!normalizedRedirect || normalizedRedirect === canonicalPath) {
-        continue
-      }
-
-      redirects.push({
-        from: normalizedRedirect,
-        to: canonicalPath,
-        source: node.path,
-      })
-    }
-  }
+        fetchFile: (filePath) => fetchRepoFile(repo, branch, filePath),
+        onCanonicalPath: (canonicalPath) => paths.add(canonicalPath),
+      }),
+  )
 
   return {
     paths: Array.from(paths),
-    redirects: buildRedirectManifest(redirects, {
+    redirects: buildRedirectManifest(redirectsByFile.flat(), {
       label: `docs redirects for ${repo}@${branch}:${docsRoot}`,
     }),
   }
@@ -219,6 +286,7 @@ async function buildDocsPathManifest({
   branch: string
   docsRoot: string
 }): Promise<DocsManifest> {
+  const { fetchApiContents } = await loadDocumentsServerModule()
   const nodes = await fetchApiContents(repo, branch, docsRoot)
 
   if (!nodes) {
@@ -242,6 +310,11 @@ export const fetchDocsManifest = createServerFn({ method: 'GET' })
   .validator(docsManifestInput)
   .handler(async ({ data }) => {
     const { repo, branch, docsRoot } = data
+    const [{ shouldUseLocalDocsFiles }, { getCachedDocsArtifact }] =
+      await Promise.all([
+        loadDocumentsServerModule(),
+        loadGitHubContentCacheServerModule(),
+      ])
 
     if (shouldUseLocalDocsFiles()) {
       return buildDocsManifest({ repo, branch, docsRoot })
@@ -262,6 +335,11 @@ export const fetchDocsPathManifest = createServerFn({ method: 'GET' })
   .validator(docsManifestInput)
   .handler(async ({ data }) => {
     const { repo, branch, docsRoot } = data
+    const [{ shouldUseLocalDocsFiles }, { getCachedDocsArtifact }] =
+      await Promise.all([
+        loadDocumentsServerModule(),
+        loadGitHubContentCacheServerModule(),
+      ])
 
     if (shouldUseLocalDocsFiles()) {
       return buildDocsPathManifest({ repo, branch, docsRoot })
@@ -281,6 +359,8 @@ export const fetchDocsPathManifest = createServerFn({ method: 'GET' })
 export const fetchDocsRedirect = createServerFn({ method: 'GET' })
   .validator(docsRedirectInput)
   .handler(async ({ data }) => {
+    const { isRecoverableGitHubContentError } =
+      await loadDocumentsServerModule()
     let manifest: DocsManifest
 
     try {
@@ -329,6 +409,7 @@ export const fetchDocs = createServerFn({ method: 'GET' })
       throw notFound()
     }
 
+    const { extractFrontMatter } = await loadDocumentsServerModule()
     const frontMatter = extractFrontMatter(file)
     const description =
       frontMatter.userDescription ?? removeMarkdown(frontMatter.excerpt ?? '')
@@ -386,7 +467,9 @@ export const fetchRepoDirectoryContents = createServerFn({
   .validator(repoDirectoryInput)
   .handler(async ({ data }: { data: RepoDirectoryRequest }) => {
     const { repo, branch, startingPath } = data
-    let githubContents: Awaited<ReturnType<typeof fetchApiContents>>
+    const { fetchApiContents, isRecoverableGitHubContentError } =
+      await loadDocumentsServerModule()
+    let githubContents: Array<GitHubFileNode> | null
 
     try {
       githubContents = await fetchApiContents(repo, branch, startingPath)