no index on md URLs

Author: lennessyy

docusaurus.config.js

@@ -389,6 +389,7 @@ module.exports = async function createConfigAsync() {
           // Generate both llms.txt (index) and llms-full.txt (complete content)
           generateLLMsTxt: true,
           generateLLMsFullTxt: true,
+          generateMarkdownFiles: true,
 
           // Exclude imported markdown partials that should not be published as standalone LLM docs.
           ignoreFiles: ['docs/cloud/references/regions/private-service.md', 'docs/cloud/references/regions/gcpregions.md'],

vercel.json

@@ -4,6 +4,17 @@
   "github": {
     "silent": true
   },
+  "headers": [
+    {
+      "source": "/(.*)\\.md",
+      "headers": [
+        {
+          "key": "X-Robots-Tag",
+          "value": "noindex, nofollow"
+        }
+      ]
+    }
+  ],
   "redirects": [
     {
       "source": "/evaluate/serverless-workers-demo",