3981: Adds link check workflow

Author: greensomes

.github/site-crawl/crawl-site.js

@@ -0,0 +1,61 @@
+import fs from "fs";
+import { parseStringPromise } from "xml2js";
+import { chromium } from "playwright";
+import fetch from "node-fetch";
+
+async function run() {
+  const sitemapUrl = "https://revenuedata.doi.gov/sitemap-0.xml";
+
+  // Download sitemap XML
+  console.log(`Fetching sitemap from: ${sitemapUrl}`);
+  const res = await fetch(sitemapUrl);
+  if (!res.ok) {
+    throw new Error(`Failed to fetch sitemap: ${res.statusText}`);
+  }
+  const xml = await res.text();
+
+  // Parse XML to extract <loc> URLs
+  const result = await parseStringPromise(xml);
+  const urls = result.urlset.url.map((u) => u.loc[0]);
+
+  console.log(`Found ${urls.length} URLs in sitemap.`);
+
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext();
+
+  const allLinks = {};
+
+  for (const url of urls) {
+    console.log(`Visiting: ${url}`);
+
+    const page = await context.newPage();
+    await page.goto(url, { waitUntil: "networkidle" });
+
+    // Extract all visible <a href> links
+    const links = await page.$$eval("a[href]", (anchors) =>
+      anchors
+        .map((a) => a.href)
+        .filter((href) => href && !href.startsWith("javascript:"))
+    );
+
+    allLinks[url] = links;
+
+    await page.close();
+  }
+
+  await browser.close();
+
+  // Save results to JSON
+  fs.writeFileSync(
+    "found-links.json",
+    JSON.stringify(allLinks, null, 2),
+    "utf-8"
+  );
+
+  console.log("Saved found links to found-links.json");
+}
+
+run().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

.github/site-crawl/export-links.js

@@ -0,0 +1,19 @@
+import fs from "fs";
+
+const data = JSON.parse(fs.readFileSync("found-links.json", "utf-8"));
+
+const allUrls = new Set();
+
+// Add each page URL
+for (const pageUrl of Object.keys(data)) {
+  allUrls.add(pageUrl);
+
+  for (const link of data[pageUrl]) {
+    allUrls.add(link);
+  }
+}
+
+const output = Array.from(allUrls).sort().join("\n");
+fs.writeFileSync("all-urls.txt", output, "utf-8");
+
+console.log(`Wrote ${allUrls.size} unique URLs to all-urls.txt`);

.github/site-crawl/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "site-crawl",
+  "version": "1.0.0",
+  "description": "Crawler script using Playwright to fetch URLs from the sitemap, render pages, and extract all discovered links for validation and analysis in CI.",
+  "author": "Jeff Schwartz (jeffrey.schwartz@onrr.gov)",
+  "type": "module",
+  "scripts": {
+    "start": "node crawl-site.js",
+    "export-links": "node export-links.js"
+  },
+  "dependencies": {
+    "playwright": "^1.44.0",
+    "xml2js": "^0.6.2",
+    "node-fetch": "^3.3.2"
+  }
+}

.github/workflows/link-check.yml

@@ -0,0 +1,57 @@
+name: Check Links
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 3 * * 0' # runs weekly on Sunday at 3am UTC
+
+jobs:
+  crawl-and-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install crawler dependencies
+        run: |
+          cd .github/link-checker
+          npm ci
+
+      - name: Run crawler
+        run: |
+          cd .github/link-checker
+          npm start
+
+      - name: Export URLs for Lychee
+        run: |
+          cd .github/link-checker
+          npm run export-links
+
+      - name: Upload found-links.json artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: found-links
+          path: .github/link-checker/found-links.json
+
+      - name: Run Lychee link checker
+        uses: lycheeverse/lychee-action@v1.10.0
+        with:
+          args: >
+            --verbose
+            --no-progress
+            --max-concurrency 10
+            .github/link-checker/all-urls.txt
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: link-check-results
+          path: |
+            .github/link-checker/found-links.json
+            .github/link-checker/all-urls.txt