Improve varlock.dev agent readiness (#1028)

* Improve varlock.dev agent readiness

- Add a Worker entrypoint that serves a short markdown 404 body to agents
  (static assets still served first; worker only runs on asset miss)
- Enrich llms.txt with a description, when-to-use guidance, install
  commands, topic bundles, and discovery links
- Emit site-wide schema.org JSON-LD (Organization, SoftwareApplication, WebSite)
- Fix homepage heading outline
- Publish MCP server card aliases (/.well-known/mcp.json, server-cards.json)
  and an ARD ai-catalog.json, generated by scripts/gen-well-known.ts
- Add ai-catalog / llms.txt link hints in head, Link headers, and robots.txt
- Add /about, /contact, /privacy pages and footer links
- Update agent skills and docs to list the new discovery endpoints

* Apply suggestion from @philmillman

* Ignore wrangler local state in website package

* Fix agent 404 content negotiation

---------

Co-authored-by: pullfrog[bot] <226033991+pullfrog[bot]@users.noreply.github.com>

Author: philmillman

bun.lock

@@ -1,10 +1,15 @@
 # build output
 dist/
+# wrangler local state (wrangler dev)
+.wrangler/
 # generated types
 .astro/
-# generated agent-skills discovery tree (see scripts/gen-agent-skills.ts)
+# generated .well-known discovery files (see scripts/gen-well-known.ts)
 public/.well-known/agent-skills/index.json
 public/.well-known/agent-skills/varlock/
+public/.well-known/mcp.json
+public/.well-known/mcp/server-cards.json
+public/.well-known/ai-catalog.json
 
 # dependencies
 node_modules/

packages/varlock-website/.gitignore

@@ -9,6 +9,7 @@ import robotsTxt from 'astro-robots-txt';
 import starlightLlmsTxt from 'starlight-llms-txt';
 import partytown from '@astrojs/partytown';
 import remarkCustomHeaderId from 'remark-custom-header-id';
+import { outdent } from 'outdent';
 
 import varlockAstroIntegration from '@varlock/astro-integration';
 import { ENV } from 'varlock/env';
@@ -130,10 +131,92 @@ export default defineConfig({
         Sidebar: '@/components/TabbedSidebar.astro',
       },
       customCss: ['@/styles/global.css'],
-      plugins: [starlightLlmsTxt()],
+      plugins: [
+        starlightLlmsTxt({
+          projectName: 'Varlock',
+          description:
+            'Varlock is an open source CLI and library for AI-safe .env files. A .env.schema declares and validates '
+            + 'environment variables, secrets are encrypted locally or loaded from external providers, and redaction plus a '
+            + 'credential proxy keep secret values away from logs, terminals, and AI agents.',
+          details: outdent`
+            When to use varlock:
+
+            - Managing .env files across development, CI, and production environments
+            - Declaring env vars in a .env.schema with decorators like @type, @required, and @sensitive
+            - Validating configuration at startup so misconfiguration fails early with clear errors
+            - Encrypting local overrides in .env.local so nothing sensitive sits in plaintext
+            - Loading secrets from 1Password, AWS, Azure, GCP, HashiCorp Vault, Infisical, Doppler, Bitwarden, and other providers via plugins
+            - Redacting secret values from logs and console output at runtime
+            - Giving AI coding agents full context on config (names, types, docs) without exposing secret values
+            - Running agents and MCP servers behind the varlock proxy so they only ever see placeholder credentials
+            - Scanning a repo for leaked secrets with \`varlock scan\`
+
+            Install:
+
+            \`\`\`bash
+            npx varlock init                 # add to a JS/TS project
+            npm install varlock              # or add the dependency directly
+            brew install dmno-dev/tap/varlock  # standalone binary via Homebrew
+            curl -sSfL https://varlock.dev/install.sh | sh -s   # standalone binary via script
+            \`\`\`
+
+            Common CLI commands: \`varlock init\`, \`varlock load\`, \`varlock run -- <cmd>\`, \`varlock encrypt\`, \`varlock scan\`, \`varlock proxy\`.
+
+            Every page on varlock.dev can be fetched as markdown by sending an \`Accept: text/markdown\` header. The docs are also searchable over MCP at https://docs.mcp.varlock.dev/mcp (streamable HTTP) and https://docs.mcp.varlock.dev/sse.
+          `,
+          optionalLinks: [
+            { label: 'GitHub repository', url: 'https://github.com/dmno-dev/varlock', description: 'source code, issues, and discussions' },
+            { label: 'npm package', url: 'https://www.npmjs.com/package/varlock' },
+            { label: 'Changelog', url: 'https://github.com/dmno-dev/varlock/blob/main/packages/varlock/CHANGELOG.md' },
+            { label: 'Docs MCP server card', url: 'https://varlock.dev/.well-known/mcp/server-card.json', description: 'MCP transports for searching these docs' },
+            { label: 'Agent skills index', url: 'https://varlock.dev/.well-known/agent-skills/index.json', description: 'SKILL.md files for coding agents' },
+            { label: 'Varlock skill (SKILL.md)', url: 'https://varlock.dev/.well-known/agent-skills/varlock/SKILL.md' },
+            { label: 'AI catalog (ARD)', url: 'https://varlock.dev/.well-known/ai-catalog.json' },
+            { label: 'Blog', url: 'https://varlock.dev/blog/' },
+            { label: 'Discord', url: 'https://chat.dmno.dev' },
+          ],
+          customSets: [
+            {
+              label: 'Getting started and CLI reference',
+              description: 'installation, core concepts, env-spec syntax, decorators, and every CLI command',
+              paths: ['getting-started/**', 'reference/**', 'env-spec/**'],
+            },
+            {
+              label: 'AI tools, MCP and proxy',
+              description: 'using varlock with coding agents, MCP servers, and the credential proxy',
+              paths: ['guides/ai-tools{,/**}', 'guides/mcp{,/**}', 'guides/proxy{,/**}'],
+            },
+            {
+              label: 'Framework and language integrations',
+              description: 'Next.js, Vite, Astro, Bun, Python, Go, Rust, Java, PHP, Docker, CI, and more',
+              paths: ['integrations/**'],
+            },
+            {
+              label: 'Secret provider plugins',
+              description: '1Password, AWS, Azure, GCP, Vault, Infisical, Doppler, Bitwarden, Kubernetes, and other providers',
+              paths: ['plugins/**'],
+            },
+            {
+              label: 'Sandboxes',
+              description: 'sandboxing recipes for running agents with the proxy',
+              paths: ['sandboxes/**'],
+            },
+          ],
+          promote: ['getting-started/**', 'reference/cli/**'],
+          exclude: ['sandboxes/**'],
+        }),
+      ],
       head: [
         // add sitemap to head for discoverability
         { tag: 'link', attrs: { rel: 'sitemap', href: '/sitemap-index.xml' } },
+        // agent discovery hints
+        { tag: 'link', attrs: { rel: 'ai-catalog', href: '/.well-known/ai-catalog.json', type: 'application/json' } },
+        {
+          tag: 'link',
+          attrs: {
+            rel: 'alternate', href: '/llms.txt', type: 'text/markdown', title: 'llms.txt',
+          },
+        },
         // Open Graph and Twitter Card defaults (page-specific values override via head)
         { tag: 'meta', attrs: { property: 'og:type', content: 'website' } },
         { tag: 'meta', attrs: { property: 'og:site_name', content: 'Varlock' } },
@@ -169,9 +252,16 @@ export default defineConfig({
         },
       ],
       transform(content) {
-        const contentSignalLine = 'Content-Signal: ai-train=no, search=yes, ai-input=no';
-        if (content.includes(contentSignalLine)) return content;
-        return `${content.trimEnd()}\n${contentSignalLine}\n`;
+        const extraLines = [
+          'Content-Signal: ai-train=no, search=yes, ai-input=no',
+          // Agentic Resource Discovery manifest
+          'Agentmap: https://varlock.dev/.well-known/ai-catalog.json',
+        ];
+        let out = content.trimEnd();
+        for (const line of extraLines) {
+          if (!out.includes(line)) out += `\n${line}`;
+        }
+        return `${out}\n`;
       },
     }),
     partytown({

packages/varlock-website/astro.config.ts

@@ -4,13 +4,15 @@
   "type": "module",
   "version": "0.0.1",
   "scripts": {
-    "gen:agent-skills": "bun run scripts/gen-agent-skills.ts",
-    "dev": "bun run gen:agent-skills && astro dev",
-    "start": "bun run gen:agent-skills && astro dev",
-    "build": "bun run gen:agent-skills && astro build",
+    "gen:well-known": "bun run scripts/gen-well-known.ts",
+    "dev": "bun run gen:well-known && astro dev",
+    "start": "bun run gen:well-known && astro dev",
+    "build": "bun run gen:well-known && astro build",
     "build:cloudflare": "turbo build --filter=@varlock/astro-integration && varlock load && turbo build",
     "preview": "astro preview",
-    "astro": "astro"
+    "astro": "astro",
+    "test": "vitest",
+    "test:ci": "vitest --run"
   },
   "dependencies": {
     "@iconify/vue": "^5.0.0",
@@ -32,6 +34,7 @@
   "devDependencies": {
     "@varlock/astro-integration": "workspace:*",
     "varlock": "workspace:*",
+    "vitest": "catalog:",
     "wrangler": "^4.93.0"
   },
   "engines": {

packages/varlock-website/package.json

@@ -4,12 +4,17 @@ Use this skill to discover machine-readable metadata published by varlock.dev.
 
 ## Discovery locations
 
-- API Catalog: `https://varlock.dev/.well-known/api-catalog`
-- MCP Server Card: `https://varlock.dev/.well-known/mcp/server-card.json`
+- Docs index for agents: `https://varlock.dev/llms.txt` (topic bundles are linked from it under `/_llms-txt/`; full and abridged dumps at `/llms-full.txt` and `/llms-small.txt`)
+- AI catalog (ARD manifest): `https://varlock.dev/.well-known/ai-catalog.json`
+- API Catalog (RFC 9727 linkset): `https://varlock.dev/.well-known/api-catalog`
+- MCP Server Card: `https://varlock.dev/.well-known/mcp/server-card.json` (aliases: `/.well-known/mcp.json`, `/.well-known/mcp/server-cards.json`)
 - Skills index: `https://varlock.dev/.well-known/agent-skills/index.json`
+- Sitemap: `https://varlock.dev/sitemap-index.xml`
 
 ## Guidance
 
-1. Start from the skills index and verify digest integrity.
-2. Follow API catalog relations to find service documentation and descriptors.
-3. Use Link response headers on the homepage for bootstrap discovery.
+1. Start from `llms.txt` for a summary of what varlock is and when to use it, then follow the topic bundle you need.
+2. Start from the skills index and verify digest integrity before loading a SKILL.md.
+3. Follow API catalog relations to find service documentation and descriptors.
+4. Use Link response headers on the homepage for bootstrap discovery.
+5. Any docs page returns markdown when requested with `Accept: text/markdown`. Missing paths return a 404 with a short markdown body listing recovery links.

packages/varlock-website/public/.well-known/agent-skills/varlock-agent-readiness/SKILL.md

@@ -1,14 +1,17 @@
 # Varlock Docs Search
 
-Use this skill to discover and answer questions about varlock documentation with the remote MCP service.
+Use this skill to discover and answer questions about varlock documentation with the remote MCP service or the llms.txt bundles.
 
 ## Endpoints
 
 - Streamable HTTP: `https://docs.mcp.varlock.dev/mcp`
 - SSE: `https://docs.mcp.varlock.dev/sse`
+- Server card: `https://varlock.dev/.well-known/mcp/server-card.json`
+- Docs index: `https://varlock.dev/llms.txt` (links topic bundles, `/llms-full.txt`, and `/llms-small.txt`)
 
 ## Guidance
 
 1. Use the MCP tool to search docs by feature keywords.
-2. Prefer official varlock.dev links in responses.
-3. Return concise, implementation-focused answers.
+2. If MCP is unavailable, read `llms.txt` and fetch the relevant topic bundle instead of the full dump.
+3. Prefer official varlock.dev links in responses.
+4. Return concise, implementation-focused answers.

packages/varlock-website/public/.well-known/agent-skills/varlock-docs-search/SKILL.md

@@ -2,7 +2,9 @@
   "$schema": "https://modelcontextprotocol.io/schemas/2025-03-26/server-card.json",
   "serverInfo": {
     "name": "Varlock docs MCP",
-    "version": "0.0.1"
+    "version": "1.0.0",
+    "description": "Search and read the varlock documentation. No authentication required.",
+    "documentationUrl": "https://varlock.dev/guides/mcp/docs-mcp/"
   },
   "transports": [
     {

packages/varlock-website/public/.well-known/mcp/server-card.json

@@ -7,9 +7,32 @@
   Link: </guides/mcp>; rel="service-doc"
   Link: </.well-known/mcp/server-card.json>; rel="describedby"
   Link: </.well-known/agent-skills/index.json>; rel="describedby"
+  Link: </.well-known/ai-catalog.json>; rel="ai-catalog"
+  Link: </llms.txt>; rel="alternate"; type="text/markdown"
+
+/llms.txt
+  Link: </.well-known/ai-catalog.json>; rel="ai-catalog"
 
 /.well-known/api-catalog
   Content-Type: application/linkset+json; charset=utf-8
+  Access-Control-Allow-Origin: *
+
+/.well-known/mcp.json
+  Content-Type: application/json; charset=utf-8
+  Access-Control-Allow-Origin: *
+
+/.well-known/mcp/*
+  Content-Type: application/json; charset=utf-8
+  Access-Control-Allow-Origin: *
+
+/.well-known/ai-catalog.json
+  Content-Type: application/json; charset=utf-8
+  Access-Control-Allow-Origin: *
+  Cache-Control: public, max-age=3600
+
+/.well-known/agent-skills/index.json
+  Access-Control-Allow-Origin: *
 
 /.well-known/agent-skills/*/SKILL.md
   Content-Type: text/markdown; charset=utf-8
+  Access-Control-Allow-Origin: *