Optimize docs for LLM navigation (AI SEO / llms.txt / markdown mirrors)

.gitignore

@@ -2,11 +2,17 @@
 .next
 node_modules
 pnpm-lock.yaml
+# Generated LLM exports (see scripts/generate-full-docs.js)
 public/full-documentation.txt
+public/llms.txt
+public/llms-full.txt
+public/**/llms-full.txt
+public/**/*.md
 .specstory/
 # SpecStory explanation file
 .specstory/.what-is-this.md
 .claude/settings.local.json
 playwright-report/
 test-results/
-blob-report/
+blob-report/
+scripts/retrofit/retrofit-log.json

components/copy-page.tsx

@@ -8,10 +8,18 @@ import DocumentIcon from './icons/document';
 import ChatGPTIcon from './icons/chatgpt';
 import AnthropicIcon from './icons/anthropic';
 
+// Path of the raw-markdown mirror generated by scripts/generate-full-docs.js
+function markdownPathFor(asPath: string): string {
+  const cleanPath = asPath.split(/[?#]/)[0].replace(/\/$/, "");
+  return cleanPath === "" ? "/index.md" : `${cleanPath}.md`;
+}
+
 const CopyPage: React.FC = () => {
   const [isOpen, setIsOpen] = useState(false);
   const [isCopied, setIsCopied] = useState(false);
   const dropdownRef = useRef<HTMLDivElement>(null);
+  const { asPath } = useRouter();
+  const mdPath = markdownPathFor(asPath);
 
   // Close dropdown when clicking outside
   useEffect(() => {
@@ -27,15 +35,19 @@ const CopyPage: React.FC = () => {
 
   const copyPageAsMarkdown = async () => {
     try {
-      // Get the current page content
-      const pageContent = document.querySelector('main')?.innerText || '';
-      const pageTitle = document.title;
-
-      // Create markdown content
-      const markdownContent = `# ${pageTitle}\n\n${pageContent}`;
-
+      // Use the generated raw-markdown mirror of this page; fall back to the
+      // rendered text if it is unavailable (e.g. in dev before generation)
+      let markdownContent: string;
+      const response = await fetch(mdPath);
+      if (response.ok) {
+        markdownContent = await response.text();
+      } else {
+        const pageContent = document.querySelector('main')?.innerText || '';
+        markdownContent = `# ${document.title}\n\n${pageContent}`;
+      }
+
       await navigator.clipboard.writeText(markdownContent);
-      
+
       // Show success feedback
       setIsCopied(true);
       setTimeout(() => {
@@ -48,21 +60,13 @@ const CopyPage: React.FC = () => {
   };
 
   const viewAsMarkdown = () => {
-    const pageContent = document.querySelector('main')?.innerText || '';
-    const pageTitle = document.title;
-    const markdownContent = `# ${pageTitle}\n\n${pageContent}`;
-
-    // Open in new window/tab
-    const blob = new Blob([markdownContent], { type: 'text/markdown' });
-    const url = URL.createObjectURL(blob);
-    window.open(url, '_blank');
-    URL.revokeObjectURL(url);
+    window.open(mdPath, '_blank');
     setIsOpen(false);
   };
 
   const openInAI = (platform: 'chatgpt' | 'claude') => {
-    const currentUrl = window.location.href;
-    const prompt = `I'm building with GenLayer - can you read this docs page ${currentUrl} so I can ask you questions about it?`;
+    const mdUrl = `${window.location.origin}${mdPath}`;
+    const prompt = `I'm building with GenLayer - can you read this docs page ${mdUrl} so I can ask you questions about it?`;
     const encodedPrompt = encodeURIComponent(prompt);
 
     const urls = {

middleware.ts

@@ -0,0 +1,30 @@
+import { NextRequest, NextResponse } from "next/server";
+
+// Content negotiation for AI agents: clients that explicitly ask for
+// `Accept: text/markdown` on a docs page get the raw-markdown mirror
+// generated by scripts/generate-full-docs.js (same content the `.md` URL
+// suffix serves). Browsers never send text/markdown, so human traffic is
+// unaffected. `Vary: Accept` keeps CDN caches from mixing the two.
+export function middleware(req: NextRequest) {
+  const { pathname } = req.nextUrl;
+  const accept = req.headers.get("accept") || "";
+
+  const isPageRoute =
+    !pathname.startsWith("/_next") &&
+    !pathname.startsWith("/api") &&
+    !/\.[a-zA-Z0-9]+$/.test(pathname);
+
+  if (isPageRoute && accept.includes("text/markdown")) {
+    const url = req.nextUrl.clone();
+    url.pathname = pathname === "/" ? "/index.md" : `${pathname.replace(/\/$/, "")}.md`;
+    const res = NextResponse.rewrite(url);
+    res.headers.set("Vary", "Accept");
+    return res;
+  }
+
+  const res = NextResponse.next();
+  if (isPageRoute) {
+    res.headers.set("Vary", "Accept");
+  }
+  return res;
+}

next.config.js

@@ -54,7 +54,7 @@ const previousRedirects = [
   { old: "/build-with-genlayer/use-cases/llm-erc20", new: "/developers/intelligent-contracts/ideas" },
   {
     old: "/advanced-features/:page*",
-    new: "/developers/intelligent-contracts/advanced-features/:page*",
+    new: "/developers/intelligent-contracts/features/:page*",
   },
   { old: "/core-concepts/intelligent-contracts", new: "/developers/intelligent-contracts/introduction" },
   {
@@ -85,7 +85,7 @@ const actualRedirects = [
   },
   {
     old: "/build-with-genlayer/intelligent-contracts/advanced-features/:page*",
-    new: "/developers/intelligent-contracts/advanced-features/:page*",
+    new: "/developers/intelligent-contracts/features/:page*",
   },
   {
     old: "/build-with-genlayer/use-cases/:page*",
@@ -148,8 +148,16 @@ const actualRedirects = [
     new: "/developers/intelligent-contracts/examples/storage",
   },
   {
-    old: "/developers/intelligent-contracts/advanced-features",
-    new: "/developers/intelligent-contracts/advanced-features/contract-to-contract-interaction",
+    old: "/developers/intelligent-contracts/advanced-features/:page*",
+    new: "/developers/intelligent-contracts/features/:page*",
+  },
+  {
+    old: "/developers/intelligent-contracts/features/contract-to-contract-interaction",
+    new: "/developers/intelligent-contracts/features/interacting-with-intelligent-contracts",
+  },
+  {
+    old: "/developers/intelligent-contracts/features/features",
+    new: "/developers/intelligent-contracts/features",
   },
   {
     old: "/developers/intelligent-contracts/testing-and-debugging",
@@ -161,11 +169,24 @@ const actualRedirects = [
   },
   {
     old: "/developers/intelligent-contracts/error-handling",
-    new: "/developers/intelligent-contracts/advanced-features/error-handling",
+    new: "/developers/intelligent-contracts/features/error-handling",
   },
 ];
 
 const nextConfig = withNextra({
+  async rewrites() {
+    return {
+      // Runs only when no real file matched, so valid .md mirrors in
+      // public/ are served directly and misses get a markdown 404 with
+      // closest-match suggestions instead of an HTML error page.
+      afterFiles: [
+        {
+          source: "/:requested(.*\\.md)",
+          destination: "/api/markdown-404?requested=:requested",
+        },
+      ],
+    };
+  },
   async redirects() {
     return [
       // Previous redirects