|
| 1 | +#!/usr/bin/env node |
| 2 | +// Regenerates src/data/language-packs.json — the site's copy of which human |
| 3 | +// languages ship a Legesher language pack, and what tier each one carries. |
| 4 | +// |
| 5 | +// WHY A GENERATED FILE, AND NOT A HAND-WRITTEN LIST |
| 6 | +// |
| 7 | +// The tier stamped on a language is a claim about how far that language has |
| 8 | +// been reviewed, and the same claim is stamped on the published dataset. A |
| 9 | +// hand-typed list on the website is how those two silently stop agreeing: the |
| 10 | +// dataset re-exports, the site does not, and the site is then advertising a |
| 11 | +// review standard the packs no longer hold. So the site's copy is derived, |
| 12 | +// once, from the artifacts that decide the answer, and the derivation is |
| 13 | +// re-runnable. |
| 14 | +// |
| 15 | +// INPUTS — both JSON, both produced by the packs monorepo, never edited here: |
| 16 | +// |
| 17 | +// 1. The language registry (`libs/i18n/legesher_i18n/languages.json`). Itself |
| 18 | +// generated from the translations database; carries each locale's name, |
| 19 | +// endonym, direction, and `status` — the tier. |
| 20 | +// 2. The canon export manifest (`manifest.json`, written beside the parquet |
| 21 | +// files by the pack exporter). Carries the counts that actually ship: |
| 22 | +// locales, pack files, vocabulary rows, interpreter versions, and the |
| 23 | +// tier spread. |
| 24 | +// |
| 25 | +// Two inputs rather than one because neither alone is sufficient: the registry |
| 26 | +// knows the languages, the manifest knows what was published about them. They |
| 27 | +// are cross-checked against each other below, so a stale pairing fails here |
| 28 | +// instead of rendering a wrong number. |
| 29 | +// |
| 30 | +// The parquet files are deliberately not read. Everything the site renders is |
| 31 | +// in the manifest, and adding a parquet dependency to a static site build to |
| 32 | +// re-derive numbers the exporter already reconciled would buy nothing. |
| 33 | +// |
| 34 | +// Both paths are required rather than guessed, because the wrong pairing is |
| 35 | +// worse than no run at all: |
| 36 | +// |
| 37 | +// npm run sync:languages -- \ |
| 38 | +// --registry <monorepo>/libs/i18n/legesher_i18n/languages.json \ |
| 39 | +// --manifest <monorepo>/build/hf-packs/manifest.json |
| 40 | + |
| 41 | +import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'; |
| 42 | +import { dirname, resolve } from 'node:path'; |
| 43 | +import { fileURLToPath } from 'node:url'; |
| 44 | + |
| 45 | +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); |
| 46 | +const OUTPUT = resolve(REPO_ROOT, 'src/data/language-packs.json'); |
| 47 | + |
| 48 | +const USAGE = |
| 49 | + 'Usage: npm run sync:languages -- \\\n' + |
| 50 | + ' --registry <monorepo>/libs/i18n/legesher_i18n/languages.json \\\n' + |
| 51 | + ' --manifest <monorepo>/build/hf-packs/manifest.json'; |
| 52 | + |
| 53 | +function parseArgs(argv) { |
| 54 | + const args = { registry: null, manifest: null }; |
| 55 | + for (let i = 0; i < argv.length; i += 2) { |
| 56 | + const flag = argv[i]?.replace(/^--/, ''); |
| 57 | + if (!(flag in args)) { |
| 58 | + throw new Error(`Unknown argument "${argv[i]}".\n${USAGE}`); |
| 59 | + } |
| 60 | + if (argv[i + 1] === undefined) throw new Error(`--${flag} needs a path.\n${USAGE}`); |
| 61 | + args[flag] = argv[i + 1]; |
| 62 | + } |
| 63 | + for (const [flag, value] of Object.entries(args)) { |
| 64 | + if (!value) throw new Error(`--${flag} is required.\n${USAGE}`); |
| 65 | + } |
| 66 | + return args; |
| 67 | +} |
| 68 | + |
| 69 | +function readJson(label, path) { |
| 70 | + const absolute = resolve(REPO_ROOT, path); |
| 71 | + try { |
| 72 | + return JSON.parse(readFileSync(absolute, 'utf8')); |
| 73 | + } catch (error) { |
| 74 | + throw new Error( |
| 75 | + `Could not read the ${label} at ${absolute}.\n` + |
| 76 | + `Point --${label} at a checkout of the packs monorepo` + |
| 77 | + (label === 'manifest' ? ', running the pack export first if needed' : '') + |
| 78 | + `.\n ${error.message}\n${USAGE}` |
| 79 | + ); |
| 80 | + } |
| 81 | +} |
| 82 | + |
| 83 | +/** |
| 84 | + * Fail with a sentence naming the field, not a stack trace about `undefined`. |
| 85 | + * |
| 86 | + * Both inputs are generated files from another repository, so the realistic |
| 87 | + * failure is being handed the wrong file rather than a corrupt one — and the |
| 88 | + * useful error says which shape was expected. |
| 89 | + */ |
| 90 | +function requireShape(condition, message) { |
| 91 | + if (!condition) throw new Error(message); |
| 92 | +} |
| 93 | + |
| 94 | +function validateInputs(registryFile, manifest) { |
| 95 | + requireShape( |
| 96 | + registryFile && typeof registryFile === 'object' && registryFile.languages, |
| 97 | + 'The registry file has no `languages` object. Expected the generated ' + |
| 98 | + 'languages.json from the packs monorepo, not another JSON file.' |
| 99 | + ); |
| 100 | + requireShape( |
| 101 | + Array.isArray(manifest?.locales), |
| 102 | + 'The manifest has no `locales` array. Expected manifest.json from a pack ' + |
| 103 | + 'export, not another JSON file.' |
| 104 | + ); |
| 105 | + requireShape( |
| 106 | + Number.isInteger(manifest?.counts?.locales), |
| 107 | + 'The manifest has no `counts.locales` integer. Expected manifest.json from a pack export.' |
| 108 | + ); |
| 109 | + requireShape( |
| 110 | + new Set(manifest.locales).size === manifest.locales.length, |
| 111 | + 'The manifest lists the same locale more than once, so its counts cannot be trusted.' |
| 112 | + ); |
| 113 | +} |
| 114 | + |
| 115 | +/** Fail loudly when the two inputs describe different exports. */ |
| 116 | +function crossCheck(registry, manifest) { |
| 117 | + const registryCodes = Object.keys(registry).sort(); |
| 118 | + const manifestCodes = [...manifest.locales].sort(); |
| 119 | + |
| 120 | + const onlyInRegistry = registryCodes.filter((c) => !manifestCodes.includes(c)); |
| 121 | + const onlyInManifest = manifestCodes.filter((c) => !registryCodes.includes(c)); |
| 122 | + if (onlyInRegistry.length || onlyInManifest.length) { |
| 123 | + throw new Error( |
| 124 | + 'Registry and export manifest name different locales — one of them is stale.\n' + |
| 125 | + ` only in registry: ${onlyInRegistry.join(', ') || '(none)'}\n` + |
| 126 | + ` only in manifest: ${onlyInManifest.join(', ') || '(none)'}` |
| 127 | + ); |
| 128 | + } |
| 129 | + |
| 130 | + if (manifest.counts.locales !== registryCodes.length) { |
| 131 | + throw new Error( |
| 132 | + `Manifest counts ${manifest.counts.locales} locales but names ${registryCodes.length}.` |
| 133 | + ); |
| 134 | + } |
| 135 | + |
| 136 | + // The tier a language carries is read from the registry below, while the |
| 137 | + // headline "all of them are experimental" line is read from the manifest. |
| 138 | + // Proving the two agree here is what lets the pages trust either one. |
| 139 | + const derived = {}; |
| 140 | + for (const code of registryCodes) { |
| 141 | + const tier = registry[code].status; |
| 142 | + if (!tier) throw new Error(`Language "${code}" carries no status in the registry.`); |
| 143 | + derived[tier] = (derived[tier] ?? 0) + 1; |
| 144 | + } |
| 145 | + const declared = manifest.tier_counts ?? {}; |
| 146 | + const tiers = [...new Set([...Object.keys(derived), ...Object.keys(declared)])].sort(); |
| 147 | + // Both sides default to 0 before comparing. A manifest is free to declare a |
| 148 | + // rung explicitly as zero, and comparing `undefined` to `0` would reject it |
| 149 | + // with the self-contradicting message "registry 0, manifest 0". |
| 150 | + const disagreement = tiers.filter((tier) => (derived[tier] ?? 0) !== (declared[tier] ?? 0)); |
| 151 | + if (disagreement.length) { |
| 152 | + throw new Error( |
| 153 | + 'Tier spread differs between the registry and the export manifest:\n' + |
| 154 | + disagreement |
| 155 | + .map((t) => ` ${t}: registry ${derived[t] ?? 0}, manifest ${declared[t] ?? 0}`) |
| 156 | + .join('\n') |
| 157 | + ); |
| 158 | + } |
| 159 | + |
| 160 | + return derived; |
| 161 | +} |
| 162 | + |
| 163 | +function main() { |
| 164 | + const args = parseArgs(process.argv.slice(2)); |
| 165 | + const registryFile = readJson('registry', args.registry); |
| 166 | + const manifest = readJson('manifest', args.manifest); |
| 167 | + validateInputs(registryFile, manifest); |
| 168 | + |
| 169 | + const registry = registryFile.languages; |
| 170 | + const tierCounts = crossCheck(registry, manifest); |
| 171 | + |
| 172 | + const languages = Object.entries(registry) |
| 173 | + .map(([code, meta]) => ({ |
| 174 | + code, |
| 175 | + // The registry key and the BCP 47 tag are not the same string for every |
| 176 | + // language — `zh` is `zh-Hans`, `no` is `nb` — and they answer different |
| 177 | + // questions. The key identifies the pack; the tag tells a browser or a |
| 178 | + // screen reader which language the text is in. Both are carried so the |
| 179 | + // page can render the key and mark up the tag. |
| 180 | + bcp47: meta.bcp47 || code, |
| 181 | + name: meta.name, |
| 182 | + native: meta.native, |
| 183 | + rtl: Boolean(meta.rtl), |
| 184 | + tier: meta.status, |
| 185 | + })) |
| 186 | + // Sorted by English name so the rendered table needs no sort of its own. |
| 187 | + .sort((a, b) => a.name.localeCompare(b.name, 'en')); |
| 188 | + |
| 189 | + const data = { |
| 190 | + _comment: |
| 191 | + 'GENERATED FILE — do not edit by hand. Regenerate with `npm run sync:languages`. ' + |
| 192 | + 'Source: the language registry and the canon export manifest in the packs monorepo.', |
| 193 | + artifact: manifest.artifact, |
| 194 | + layer: manifest.layer, |
| 195 | + license: manifest.declared_license, |
| 196 | + counts: { |
| 197 | + locales: manifest.counts.locales, |
| 198 | + packFiles: manifest.counts.pack_files, |
| 199 | + vocabularyRows: manifest.counts.vocabulary_rows, |
| 200 | + pythonVersions: manifest.counts.python_versions, |
| 201 | + }, |
| 202 | + tierCounts, |
| 203 | + languages, |
| 204 | + }; |
| 205 | + |
| 206 | + mkdirSync(dirname(OUTPUT), { recursive: true }); |
| 207 | + writeFileSync(OUTPUT, `${JSON.stringify(data, null, 2)}\n`, 'utf8'); |
| 208 | + |
| 209 | + const spread = Object.entries(tierCounts) |
| 210 | + .map(([tier, count]) => `${count} ${tier}`) |
| 211 | + .join(', '); |
| 212 | + console.log( |
| 213 | + `Wrote ${OUTPUT}\n` + |
| 214 | + ` ${data.counts.locales} languages (${spread}), ` + |
| 215 | + `${data.counts.packFiles} pack files, ` + |
| 216 | + `${data.counts.vocabularyRows.toLocaleString('en-US')} rows` |
| 217 | + ); |
| 218 | +} |
| 219 | + |
| 220 | +try { |
| 221 | + main(); |
| 222 | +} catch (error) { |
| 223 | + console.error(error.message); |
| 224 | + process.exit(1); |
| 225 | +} |
0 commit comments