Add Google News sitemap for media news (#1383)

* Add Google News sitemap for media news

* Keep Google News sitemap populated between releases

Author: h4rkl

apps/media/__tests__/google-news-sitemap.test.ts

@@ -0,0 +1,106 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("server-only", () => ({}));
+
+vi.mock("@/lib/reader", () => ({
+  reader: {
+    collections: {
+      posts: {
+        list: vi.fn(),
+        read: vi.fn(),
+      },
+    },
+  },
+}));
+
+import { reader } from "@/lib/reader";
+import { getGoogleNewsSitemapResponse } from "@/lib/google-news-sitemap";
+
+const mockReader = reader as any;
+
+describe("getGoogleNewsSitemapResponse", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("includes only published posts from the last 48 hours", async () => {
+    mockReader.collections.posts.list.mockResolvedValue([
+      "fresh-post",
+      "stale-post",
+      "draft-post",
+      "future-post",
+    ]);
+
+    mockReader.collections.posts.read.mockImplementation((slug: string) => {
+      switch (slug) {
+        case "fresh-post":
+          return Promise.resolve({
+            status: "published",
+            title: "Fresh Solana News",
+            publishedAt: "2026-04-13T13:00:00.000Z",
+          });
+        case "stale-post":
+          return Promise.resolve({
+            status: "published",
+            title: "Old Solana News",
+            publishedAt: "2026-04-11T11:59:59.000Z",
+          });
+        case "draft-post":
+          return Promise.resolve({
+            status: "draft",
+            title: "Draft Solana News",
+            publishedAt: "2026-04-14T08:00:00.000Z",
+          });
+        case "future-post":
+          return Promise.resolve({
+            status: "published",
+            title: "Future Solana News",
+            publishedAt: "2026-04-15T08:00:00.000Z",
+          });
+        default:
+          return Promise.resolve(null);
+      }
+    });
+
+    const response = await getGoogleNewsSitemapResponse(
+      new Date("2026-04-14T12:00:00.000Z"),
+    );
+    const xml = await response.text();
+
+    expect(response.status).toBe(200);
+    expect(response.headers.get("Content-Type")).toContain("application/xml");
+    expect(xml).toContain(
+      '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">',
+    );
+    expect(xml).toContain("<loc>https://solana.com/news/fresh-post</loc>");
+    expect(xml).toContain("<news:name>Solana</news:name>");
+    expect(xml).toContain("<news:language>en</news:language>");
+    expect(xml).toContain(
+      "<news:publication_date>2026-04-13T13:00:00.000Z</news:publication_date>",
+    );
+    expect(xml).toContain("<news:title>Fresh Solana News</news:title>");
+    expect(xml).toContain("<loc>https://solana.com/news/stale-post</loc>");
+    expect(xml).toContain("<lastmod>2026-04-11T11:59:59.000Z</lastmod>");
+    expect(xml).not.toContain("<news:title>Old Solana News</news:title>");
+    expect(xml).not.toContain("draft-post");
+    expect(xml).not.toContain("future-post");
+  });
+
+  it("escapes XML-sensitive characters in titles", async () => {
+    mockReader.collections.posts.list.mockResolvedValue(["escaped-post"]);
+    mockReader.collections.posts.read.mockResolvedValue({
+      status: "published",
+      title: 'AT&T <Solana> "News"',
+      publishedAt: "2026-04-14T10:00:00.000Z",
+    });
+
+    const response = await getGoogleNewsSitemapResponse(
+      new Date("2026-04-14T12:00:00.000Z"),
+    );
+    const xml = await response.text();
+
+    expect(xml).toContain(
+      "<news:title>AT&amp;T &lt;Solana&gt; &quot;News&quot;</news:title>",
+    );
+  });
+});

apps/media/app/news/google-news.xml/route.ts

@@ -0,0 +1,7 @@
+import { getGoogleNewsSitemapResponse } from "@/lib/google-news-sitemap";
+
+export const revalidate = 300;
+
+export async function GET() {
+  return getGoogleNewsSitemapResponse();
+}

apps/media/lib/google-news-sitemap.ts

@@ -0,0 +1,129 @@
+import { NextResponse } from "next/server";
+import { reader } from "@/lib/reader";
+import { isPublishedPost } from "@/lib/keystatic/post-status";
+import { parsePublishedAt } from "@/lib/keystatic/publishing";
+
+const BASE_URL = "https://solana.com";
+const NEWS_URL = `${BASE_URL}/news`;
+const GOOGLE_NEWS_SITEMAP_CANONICAL_PATH = "/news/google-news.xml";
+const PUBLICATION_NAME = "Solana";
+const PUBLICATION_LANGUAGE = "en";
+const GOOGLE_NEWS_LOOKBACK_HOURS = 48;
+const MAX_NEWS_URLS = 1000;
+const XML_CONTENT_TYPE = "application/xml; charset=utf-8";
+const XML_CACHE_CONTROL = "public, s-maxage=300, stale-while-revalidate=600";
+
+export const GOOGLE_NEWS_SITEMAP_CANONICAL_URL = `${BASE_URL}${GOOGLE_NEWS_SITEMAP_CANONICAL_PATH}`;
+
+function escapeXml(value: string): string {
+  return value
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&apos;");
+}
+
+type RecentPublishedPost = {
+  slug: string;
+  title: string;
+  publishedAt: Date;
+};
+
+async function getPublishedPosts(): Promise<RecentPublishedPost[]> {
+  const allSlugs = await reader.collections.posts.list();
+  const posts: RecentPublishedPost[] = [];
+
+  for (const slug of allSlugs) {
+    const post = await reader.collections.posts.read(slug);
+    if (!isPublishedPost(post)) {
+      continue;
+    }
+
+    const publishedAt = parsePublishedAt(post.publishedAt);
+    if (!publishedAt) {
+      continue;
+    }
+
+    posts.push({
+      slug,
+      title: String(post.title || slug),
+      publishedAt,
+    });
+  }
+
+  posts.sort(
+    (left, right) => right.publishedAt.getTime() - left.publishedAt.getTime(),
+  );
+
+  return posts.slice(0, MAX_NEWS_URLS);
+}
+
+async function buildGoogleNewsSitemapXml(
+  now: Date = new Date(),
+): Promise<string> {
+  const minNewsPublishedAt = new Date(
+    now.getTime() - GOOGLE_NEWS_LOOKBACK_HOURS * 60 * 60 * 1000,
+  );
+  const posts = await getPublishedPosts();
+  const urls = posts.map((post) => {
+    const loc = `${NEWS_URL}/${post.slug}`;
+    const isGoogleNewsEligible =
+      post.publishedAt >= minNewsPublishedAt && post.publishedAt <= now;
+
+    const lines = [
+      "<url>",
+      `<loc>${escapeXml(loc)}</loc>`,
+      `<lastmod>${escapeXml(post.publishedAt.toISOString())}</lastmod>`,
+    ];
+
+    if (isGoogleNewsEligible) {
+      lines.push(
+        "<news:news>",
+        "<news:publication>",
+        `<news:name>${escapeXml(PUBLICATION_NAME)}</news:name>`,
+        `<news:language>${escapeXml(PUBLICATION_LANGUAGE)}</news:language>`,
+        "</news:publication>",
+        `<news:publication_date>${escapeXml(post.publishedAt.toISOString())}</news:publication_date>`,
+        `<news:title>${escapeXml(post.title)}</news:title>`,
+        "</news:news>",
+      );
+    }
+
+    lines.push("</url>");
+
+    return lines.join("");
+  });
+
+  return [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">',
+    ...urls,
+    "</urlset>",
+    "",
+  ].join("\n");
+}
+
+export async function getGoogleNewsSitemapResponse(
+  now: Date = new Date(),
+): Promise<NextResponse> {
+  try {
+    const xml = await buildGoogleNewsSitemapXml(now);
+
+    return new NextResponse(xml, {
+      status: 200,
+      headers: {
+        "Content-Type": XML_CONTENT_TYPE,
+        "Cache-Control": XML_CACHE_CONTROL,
+      },
+    });
+  } catch (error) {
+    console.error("Error generating Google News sitemap:", error);
+    return new NextResponse("Error generating Google News sitemap", {
+      status: 500,
+      headers: {
+        "Content-Type": "text/plain; charset=utf-8",
+      },
+    });
+  }
+}

apps/web/src/app/robots.ts

@@ -8,6 +8,9 @@ export default function robots(): MetadataRoute.Robots {
       userAgent: "*",
       allow: "/",
     },
-    sitemap: "https://solana.com/sitemap.xml",
+    sitemap: [
+      "https://solana.com/sitemap.xml",
+      "https://solana.com/news/google-news.xml",
+    ],
   };
 }