fix(seo): noindex auth and docs crawl #2954
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| import type { MetadataRoute } from "next"; | ||
|
|
||
| export default function robots(): MetadataRoute.Robots { | ||
| const docsBaseUrl = process.env.DOCS_URL ?? "https://docs.llmgateway.io"; | ||
|
|
||
| return { | ||
| rules: [ | ||
| { | ||
| userAgent: "*", | ||
| allow: "/", | ||
| disallow: ["/api/", "/llms.mdx/", "/docs-og/"], | ||
|
smakosh marked this conversation as resolved.
Outdated
|
||
| }, | ||
| ], | ||
| sitemap: `${docsBaseUrl}/sitemap.xml`, | ||
| }; | ||
|
Comment on lines
+10
to
+25
Contributor
There was a problem hiding this comment. 🩺 Stability & Availability | 🟡 Minor | ⚡ Quick win 🧩 Analysis chain🏁 Script executed: #!/bin/bash
# Check if other files in the docs app use force-dynamic alongside env var reads
rg -n "force-dynamic|dynamic.*=.*force" apps/docs/ --type=ts -C2Repository: theopenco/llmgateway Length of output: 418 🏁 Script executed: #!/bin/bash
set -euo pipefail
# Inspect the docs app files that read DOCS_URL and the robots route itself.
sed -n '1,120p' apps/docs/app/layout.tsx
printf '\n---\n'
sed -n '1,120p' apps/docs/app/robots.ts
printf '\n---\n'
rg -n "DOCS_URL|dynamic = \"force-dynamic\"" apps/docs/app -C 2Repository: theopenco/llmgateway Length of output: 3605 🏁 Script executed: #!/bin/bash
set -euo pipefail
# Read the robots route and nearby app route conventions.
sed -n '1,120p' apps/docs/app/robots.ts
printf '\n---\n'
sed -n '1,80p' apps/docs/app/layout.tsx
printf '\n---\n'
rg -n "export const dynamic|MetadataRoute.Robots|DOCS_URL" apps/docs/app -C 2Repository: theopenco/llmgateway Length of output: 3674 🏁 Script executed: #!/bin/bash
set -euo pipefail
sed -n '1,120p' apps/docs/app/sitemap.tsRepository: theopenco/llmgateway Length of output: 685 Add 🤖 Prompt for AI Agents |
||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| import { source } from "@/lib/source"; | ||
|
|
||
| import type { MetadataRoute } from "next"; | ||
|
|
||
| export default function sitemap(): MetadataRoute.Sitemap { | ||
| const docsBaseUrl = process.env.DOCS_URL ?? "https://docs.llmgateway.io"; | ||
| const buildDate = new Date(); | ||
|
|
||
| return source.getPages().map((page) => { | ||
| const path = page.url === "/" ? "" : page.url; | ||
| return { | ||
| url: `${docsBaseUrl}${path}`, | ||
| lastModified: buildDate, | ||
|
smakosh marked this conversation as resolved.
Outdated
|
||
| changeFrequency: page.url === "/" ? "weekly" : "monthly", | ||
| priority: page.url === "/" ? 1 : 0.7, | ||
| }; | ||
| }); | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,88 @@ | ||
| import Link from "next/link"; | ||
| import { redirect } from "next/navigation"; | ||
|
|
||
| import { HeroRSC } from "@/components/landing/hero-rsc"; | ||
|
|
||
| import type { Metadata } from "next"; | ||
|
|
||
| function slugify(label: string) { | ||
| return label | ||
| .toLowerCase() | ||
| .replace(/[^a-z0-9]+/g, "-") | ||
| .replace(/(^-|-$)/g, ""); | ||
| } | ||
|
|
||
| export const metadata: Metadata = { | ||
| title: "Blog Categories", | ||
| description: | ||
| "Browse LLM Gateway blog posts by category — product updates, tutorials, deep-dives, and more.", | ||
| alternates: { canonical: "/blog/category" }, | ||
| openGraph: { | ||
| title: "Blog Categories", | ||
| description: | ||
| "Browse LLM Gateway blog posts by category — product updates, tutorials, deep-dives, and more.", | ||
| url: "https://llmgateway.io/blog/category", | ||
| type: "website", | ||
| }, | ||
| }; | ||
|
|
||
| export default async function BlogCategoryIndexPage() { | ||
| const { allBlogs } = await import("content-collections"); | ||
|
smakosh marked this conversation as resolved.
Outdated
|
||
|
|
||
| const categories = new Map<string, string>(); | ||
| for (const post of allBlogs) { | ||
| if (post.draft) { | ||
| continue; | ||
| } | ||
| for (const category of post.categories ?? []) { | ||
| const slug = slugify(category); | ||
| if (!categories.has(slug)) { | ||
| categories.set(slug, category); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| const sorted = Array.from(categories.entries()).sort((a, b) => | ||
| a[1].localeCompare(b[1]), | ||
| ); | ||
|
|
||
| if (sorted.length === 1) { | ||
| redirect(`/blog/category/${encodeURIComponent(sorted[0][0])}`); | ||
|
smakosh marked this conversation as resolved.
Outdated
|
||
| } | ||
|
|
||
| return ( | ||
| <> | ||
| <HeroRSC navbarOnly /> | ||
| <div className="container mx-auto max-w-3xl px-4 py-16 pt-28 md:pt-36"> | ||
| <h1 className="mb-4 text-3xl font-bold tracking-tight md:text-4xl"> | ||
| Blog Categories | ||
| </h1> | ||
| <p className="mb-10 text-muted-foreground"> | ||
| Browse posts by topic across the LLM Gateway blog. | ||
| </p> | ||
| {sorted.length === 0 ? ( | ||
| <p className="text-muted-foreground"> | ||
| No categories yet.{" "} | ||
| <Link href="/blog" className="text-primary underline"> | ||
| View all posts | ||
| </Link> | ||
| . | ||
| </p> | ||
| ) : ( | ||
| <ul className="grid gap-3 sm:grid-cols-2"> | ||
| {sorted.map(([slug, label]) => ( | ||
| <li key={slug}> | ||
| <Link | ||
| href={`/blog/category/${encodeURIComponent(slug)}`} | ||
| className="block rounded-lg border border-border bg-card px-4 py-3 text-sm font-medium transition-colors hover:bg-muted" | ||
| > | ||
| {label} | ||
| </Link> | ||
| </li> | ||
| ))} | ||
| </ul> | ||
| )} | ||
| </div> | ||
| </> | ||
| ); | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.