DC-5820 AI Agents Served Markdown (#7237)

* ai crawler check successful

* ai crawlers checked

* update

* added anthropic

* middleware update

* improve detection

---------

Co-authored-by: Mike Hartington <mikehartington@gmail.com>

Author: aidankmcalister

docusaurus.config.ts

@@ -4,6 +4,7 @@ import { themes as prismThemes } from "prism-react-renderer";
 
 import type { Config, RouteConfig } from "@docusaurus/types";
 import type * as Preset from "@docusaurus/preset-classic";
+import type { PluginOptions } from "@signalwire/docusaurus-plugin-llms-txt/public";
 
 const DOCUSAURUS_BASE_URL = process.env.DOCUSAURUS_BASE_URL ?? "/";
 
@@ -91,6 +92,21 @@ const config: Config = {
         enableInDevelopment: false,
       },
     ],
+    [
+      "@signalwire/docusaurus-plugin-llms-txt",
+      {
+        // Enable with defaults
+        generate: {
+          enableMarkdownFiles: true,
+          enableLlmsFullTxt: false,
+        },
+        include: {
+          includeBlog: false,
+          includePages: false,
+          includeDocs: true,
+        },
+      } satisfies PluginOptions,
+    ],
     async function pluginLlmsTxt(context) {
       return {
         name: "llms-txt-plugin",
@@ -297,31 +313,31 @@ const config: Config = {
               to: "/postgres/database/prisma-studio",
               label: "Studio",
               sub: "Explore and manipulate your data",
-              icon: "fa-regular fa-table"
+              icon: "fa-regular fa-table",
             },
             {
               type: "docSidebar",
               sidebarId: "optimizeSidebar",
               className: "teal",
               label: "Optimize",
               sub: "AI-driven query analysis",
-              icon: "fa-regular fa-magnifying-glass-chart"
+              icon: "fa-regular fa-magnifying-glass-chart",
             },
             {
               type: "docSidebar",
               sidebarId: "accelerateSidebar",
               className: "teal",
               label: "Accelerate",
               sub: "Make your database global",
-              icon: "fa-regular fa-bolt"
+              icon: "fa-regular fa-bolt",
             },
             {
               type: "docSidebar",
               className: "teal",
               sidebarId: "aiSidebar",
               label: "Prisma + AI",
               sub: "Build faster with Prisma + AI",
-              icon: "fa-regular fa-robot"
+              icon: "fa-regular fa-robot",
             },
           ],
         },

functions/_middleware.ts

@@ -0,0 +1,113 @@
+interface Env {
+  ASSETS: {
+    fetch: typeof fetch;
+  };
+}
+
+const AI_CRAWLER_PATTERNS = [
+  'GPTBot',
+  'OAI-SearchBot',
+  'ChatGPT-User',
+  'ChatGPT-User 2.0',
+  'anthropic-ai',
+  'OpenAI',
+  'ClaudeBot',
+  'claude-web',
+  'Claude/',
+  'Claude-User',
+  'anthropic',
+  'PerplexityBot',
+  'Perplexity-User',
+  'Google-Extended',
+  'BingBot',
+  'Amazonbot',
+  'Applebot',
+  'Applebot-Extended',
+  'FacebookBot',
+  'meta-externalagent',
+  'LinkedInBot',
+  'Bytespider',
+  'AI2Bot',
+  'CCBot',
+  'Diffbot',
+  'DuckAssistBot',
+  'DuckDuckGo',
+  'DeepSeekBot',
+  'cohere-ai',
+  'omgili',
+  'TimpiBot',
+  'YouBot',
+  'MistralAI-User',
+  'ImagesiftBot',
+  'Scrapy',
+
+  'chatgpt',
+  'openai',
+  'gpt',
+  'claude',
+  'anthropic',
+  'cursor',
+  'windsurf',
+  'perplexity',
+  'copilot',
+  'ai-agent',
+  'llm-agent',
+];
+
+function isAICrawler(request: Request): boolean {
+  // if (!userAgent) return false;
+
+  const userAgent = request.headers.get('user-agent') || '';
+  const accept = request.headers.get('accept') || '';
+
+  const hasHtml = accept.includes('text/html');
+
+  const prefersNonHtml =
+    !hasHtml &&
+    (accept.includes('application/json') ||
+      accept.includes('text/plain') ||
+      accept.includes('application/xml'));
+
+  const hasAiUserAgent = AI_CRAWLER_PATTERNS.some((pattern) => userAgent.toLowerCase().includes(pattern.toLowerCase()));
+
+  return prefersNonHtml || hasAiUserAgent
+
+  // const normalizedUA = userAgent.toLowerCase();
+  // return AI_CRAWLER_PATTERNS.some((pattern) => normalizedUA.includes(pattern.toLowerCase()));
+}
+
+export const onRequest: PagesFunction<Env> = async (context) => {
+  const { request, next } = context;
+
+  if (isAICrawler(request)) {
+    const url = new URL(request.url);
+    const urlPath = url.pathname;
+
+    const markdownPath = `${urlPath}.md`;
+
+    const markdownRequest = new Request(new URL(markdownPath, url.origin), {
+      method: request.method,
+      headers: request.headers,
+    });
+
+    const response = await context.env.ASSETS.fetch(markdownRequest);
+
+    if (response.ok) {
+      // Check what content type we actually got
+      const actualContentType = response.headers.get('content-type');
+      console.log(`Fetched ${markdownPath}, got content-type: ${actualContentType}`);
+
+      return new Response(response.body, {
+        status: 200,
+        headers: {
+          'Content-Type': 'text/markdown; charset=utf-8',
+          'Cache-Control': 'public, max-age=3600',
+        },
+      });
+    }
+
+    return next();
+  }
+
+  return next();
+};

functions/types.d.ts

@@ -0,0 +1,12 @@
+// TypeScript definitions for Cloudflare Pages Functions
+interface EventContext<Env = unknown> {
+  request: Request;
+  env: Env;
+  params: Record<string, string>;
+  data: Record<string, unknown>;
+  next: () => Promise<Response>;
+  waitUntil: (promise: Promise<unknown>) => void;
+  passThroughOnException: () => void;
+}
+
+type PagesFunction<Env = unknown> = (context: EventContext<Env>) => Response | Promise<Response>;