|
| 1 | +// test_a11y_desktop/test_a11y_mobile in .circleci/config.yml already run with |
| 2 | +// parallelism: 2, but before this script existed both containers ran the |
| 3 | +// exact same full sitemap through pa11y-ci — parallelism was configured but |
| 4 | +// actually scans of all URLs in duplicate. This script gives |
| 5 | +// each container a distinct slice of the sitemap so the parallelism is real. |
| 6 | +import fs from "node:fs"; |
| 7 | +import path from "node:path"; |
| 8 | +import http from "node:http"; |
| 9 | +import https from "node:https"; |
| 10 | + |
| 11 | +// Same exclusion pa11y-ci:sitemap/-mobile already pass via |
| 12 | +// `--sitemap-exclude '/*.pdf|next/'`, kept in sync so sharded runs exclude |
| 13 | +// the same URLs as the unsharded ones. |
| 14 | +const EXCLUDE_PATTERN = new RegExp("/*.pdf|next/"); |
| 15 | + |
| 16 | +function parseArgs(argv) { |
| 17 | + const args = {}; |
| 18 | + for (let i = 0; i < argv.length; i += 2) { |
| 19 | + const key = argv[i].replace(/^--/, ""); |
| 20 | + args[key] = argv[i + 1]; |
| 21 | + } |
| 22 | + return args; |
| 23 | +} |
| 24 | + |
| 25 | +function fetchText(url) { |
| 26 | + const client = url.startsWith("https:") ? https : http; |
| 27 | + return new Promise((resolve, reject) => { |
| 28 | + client |
| 29 | + .get(url, (res) => { |
| 30 | + if (res.statusCode < 200 || res.statusCode >= 300) { |
| 31 | + reject(new Error(`Failed to fetch ${url}: HTTP ${res.statusCode}`)); |
| 32 | + res.resume(); |
| 33 | + return; |
| 34 | + } |
| 35 | + let body = ""; |
| 36 | + res.setEncoding("utf8"); |
| 37 | + res.on("data", (chunk) => (body += chunk)); |
| 38 | + res.on("end", () => resolve(body)); |
| 39 | + }) |
| 40 | + .on("error", reject); |
| 41 | + }); |
| 42 | +} |
| 43 | + |
| 44 | +// The sitemap is fetched over the network, so its <loc> entries are |
| 45 | +// untrusted input: validate each one resolves to a well-formed http(s) URL |
| 46 | +// on the same origin as the sitemap itself before it's allowed anywhere near |
| 47 | +// the shard config that gets written to disk and fed to pa11y-ci. Jekyll's |
| 48 | +// sitemap emits root-relative paths (e.g. "/components/button/"), so each |
| 49 | +// entry is resolved against sitemapUrl as a base. |
| 50 | +function extractUrls(sitemapXml, sitemapUrl) { |
| 51 | + const allowedOrigin = new URL(sitemapUrl).origin; |
| 52 | + const urls = []; |
| 53 | + const locRegex = /<loc>(.*?)<\/loc>/g; |
| 54 | + let match; |
| 55 | + while ((match = locRegex.exec(sitemapXml)) !== null) { |
| 56 | + const raw = match[1].trim(); |
| 57 | + let parsed; |
| 58 | + try { |
| 59 | + parsed = new URL(raw, sitemapUrl); |
| 60 | + } catch { |
| 61 | + console.warn(`Skipping malformed sitemap URL: ${raw}`); |
| 62 | + continue; |
| 63 | + } |
| 64 | + if ( |
| 65 | + (parsed.protocol !== "http:" && parsed.protocol !== "https:") || |
| 66 | + parsed.origin !== allowedOrigin |
| 67 | + ) { |
| 68 | + console.warn(`Skipping out-of-origin sitemap URL: ${raw}`); |
| 69 | + continue; |
| 70 | + } |
| 71 | + urls.push(parsed.href); |
| 72 | + } |
| 73 | + return urls; |
| 74 | +} |
| 75 | + |
| 76 | +// Round-robin rather than contiguous slices: sitemap URLs cluster by |
| 77 | +// directory (components, templates, patterns, ...), so a contiguous split |
| 78 | +// would risk loading all the heavy pages onto one shard. Interleaving |
| 79 | +// spreads page types evenly across containers without needing per-page |
| 80 | +// timing data. |
| 81 | +function shardUrls(urls, index, total) { |
| 82 | + return urls.filter((_, i) => i % total === index); |
| 83 | +} |
| 84 | + |
| 85 | +async function main() { |
| 86 | + const args = parseArgs(process.argv.slice(2)); |
| 87 | + const sitemapUrl = args.sitemap || "http://localhost:4000/sitemap.xml"; |
| 88 | + const baseConfigPath = args["base-config"] || ".pa11yci"; |
| 89 | + const outPath = args.out; |
| 90 | + |
| 91 | + if (!outPath) { |
| 92 | + throw new Error("--out <path> is required"); |
| 93 | + } |
| 94 | + |
| 95 | + // CircleCI sets these automatically from a job's `parallelism:` value. |
| 96 | + const index = parseInt(process.env.CIRCLE_NODE_INDEX || "0", 10); |
| 97 | + const total = parseInt(process.env.CIRCLE_NODE_TOTAL || "1", 10); |
| 98 | + |
| 99 | + const sitemapXml = await fetchText(sitemapUrl); |
| 100 | + const allUrls = extractUrls(sitemapXml, sitemapUrl).filter( |
| 101 | + (url) => !EXCLUDE_PATTERN.test(url) |
| 102 | + ); |
| 103 | + const shard = shardUrls(allUrls, index, total); |
| 104 | + |
| 105 | + const baseConfig = JSON.parse( |
| 106 | + fs.readFileSync(path.resolve(baseConfigPath), "utf8") |
| 107 | + ); |
| 108 | + |
| 109 | + // Only `defaults` carries over from the base .pa11yci/.pa11yci--mobile — |
| 110 | + // `urls` here is this shard's slice, not whatever (if anything) was in |
| 111 | + // the base config. |
| 112 | + const shardConfig = { |
| 113 | + defaults: baseConfig.defaults, |
| 114 | + urls: shard, |
| 115 | + }; |
| 116 | + |
| 117 | + fs.writeFileSync(outPath, JSON.stringify(shardConfig, null, 2)); |
| 118 | + |
| 119 | + console.log( |
| 120 | + `Shard ${index + 1}/${total}: ${shard.length}/${allUrls.length} URLs ` + |
| 121 | + `(from ${sitemapUrl}) written to ${outPath}` |
| 122 | + ); |
| 123 | +} |
| 124 | + |
| 125 | +main().catch((err) => { |
| 126 | + console.error(err); |
| 127 | + process.exit(1); |
| 128 | +}); |
0 commit comments