fix(styles): embedded exact-parity sweep, wave 2 #2828
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fidelity | |
| permissions: | |
| contents: read | |
| on: | |
| push: | |
| branches: ["main"] | |
| pull_request: | |
| paths: | |
| - "crates/citum-engine/**" | |
| - "crates/citum-migrate/**" | |
| - "scripts/oracle*.js" | |
| - "scripts/report-core.js" | |
| - "scripts/check-core-quality.js" | |
| - "scripts/check-core-quality.test.js" | |
| - "scripts/check-oracle-regression.js" | |
| - "scripts/report-data/embedded-parity-baseline.json" | |
| - "scripts/report-data/parity-adjudication.json" | |
| - "scripts/lib/**" | |
| - "styles/**" | |
| - "tests/**" | |
| - ".github/workflows/fidelity.yml" | |
| - ".github/workflows/compat-report.yml" | |
| schedule: | |
| - cron: "0 5 * * *" | |
| jobs: | |
| fidelity: | |
| name: Fidelity Checks | |
| runs-on: ubuntu-latest | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUSTFLAGS: "-C link-arg=-fuse-ld=mold" | |
| CARGO_INCREMENTAL: "0" | |
| SCCACHE_DIR: /home/runner/.cache/sccache | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: Determine fidelity scope | |
| id: fidelity-scope | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| BASE_REF: ${{ github.base_ref }} | |
| run: | | |
| if [ "$EVENT_NAME" = "pull_request" ]; then | |
| changed="$(git diff --name-only "origin/$BASE_REF...HEAD")" | |
| else | |
| changed="$(git diff --name-only HEAD^ HEAD 2>/dev/null || git diff --name-only HEAD)" | |
| fi | |
| broad="$(printf '%s\n' "$changed" | grep -E '^(crates/citum-engine/|crates/citum-migrate/|scripts/oracle.*\.js$|scripts/report-core\.js$|scripts/check-core-quality\.js$|scripts/check-oracle-regression\.js$|scripts/lib/|tests/)' || true)" | |
| style_files="$(printf '%s\n' "$changed" | grep -E '^styles/(embedded/)?[^/]+\.yaml$' || true)" | |
| if [ -n "$style_files" ] && [ -z "$broad" ]; then | |
| # report-core.js's SKIPPED_STYLES excludes these from its corpus | |
| # entirely (no CSL oracle, or hidden inheritance roots) — passing | |
| # one to --styles crashes with "Unknown style name(s)". Keep this | |
| # list in sync with SKIPPED_STYLES in scripts/report-core.js. | |
| styles="$(printf '%s\n' "$style_files" \ | |
| | sed -E 's#^styles/(embedded/)?##; s#\.yaml$##' \ | |
| | grep -vE '^(alpha|chicago-18-base|chicago-notes-18th-script)$' \ | |
| | sort -u | paste -sd, -)" | |
| else | |
| styles="" | |
| fi | |
| if [ -n "$styles" ]; then | |
| echo "mode=selected" >> "$GITHUB_OUTPUT" | |
| echo "styles=$styles" >> "$GITHUB_OUTPUT" | |
| echo "Running fidelity for changed styles: $styles" | |
| else | |
| echo "mode=all" >> "$GITHUB_OUTPUT" | |
| echo "styles=" >> "$GITHUB_OUTPUT" | |
| echo "Running full fidelity suite" | |
| fi | |
| - name: Initialize optional corpora | |
| run: git submodule update --init --depth 1 styles-legacy tests/csl-test-suite | |
| - name: Set up Rust | |
| uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| - name: Install mold linker and sccache | |
| run: sudo apt-get update && sudo apt-get install -y mold sccache | |
| - name: Set up Node | |
| uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 | |
| with: | |
| node-version: "22" | |
| - name: Cache Rust (dependencies + sccache) | |
| uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| shared-key: "citum-core-rust" | |
| cache-workspace-crates: true | |
| cache-directories: /home/runner/.cache/sccache | |
| # Restored target caches can contain v8 build metadata without the | |
| # downloaded native archive; rebuild only this package's artifacts. | |
| - name: Invalidate cached rusty_v8 build artifacts | |
| run: cargo clean -p v8 | |
| - name: Configure environment | |
| run: | | |
| echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" | |
| sccache --start-server | |
| - name: Install script dependencies | |
| run: npm install --prefix scripts | |
| - name: Run corpus-aware script tests | |
| if: ${{ steps.fidelity-scope.outputs.mode == 'all' }} | |
| run: node --test --test-timeout=300000 scripts/oracle.test.js scripts/report-core.test.js scripts/check-core-quality.test.js scripts/oracle-yaml.test.js | |
| - name: Check core fidelity and SQI drift | |
| run: | | |
| if [ "${{ steps.fidelity-scope.outputs.mode }}" = "selected" ]; then | |
| node scripts/report-core.js \ | |
| --all-features \ | |
| --styles "${{ steps.fidelity-scope.outputs.styles }}" \ | |
| > /tmp/core-report.json | |
| node <<'NODE' | |
| const fs = require('fs'); | |
| const report = JSON.parse(fs.readFileSync('/tmp/core-report.json', 'utf8')); | |
| const styles = Array.isArray(report.styles) ? report.styles : []; | |
| if (styles.length === 0) { | |
| console.error('Selected fidelity report has no styles'); | |
| process.exit(1); | |
| } | |
| const styleMap = new Map(styles.map((s) => [s.name, s])); | |
| // A style-YAML-only PR is exactly the kind of change that moves | |
| // fidelity and exact parity, so any changed style that is subject to | |
| // either hard gate must be checked here, not just reported. Styles | |
| // outside both baselines are printed for visibility only, matching | |
| // the full-corpus gate's scope (see check-core-quality.js). | |
| const fidelityBaseline = JSON.parse( | |
| fs.readFileSync('scripts/report-data/core-quality-baseline.json', 'utf8') | |
| ); | |
| const parityBaseline = JSON.parse( | |
| fs.readFileSync('scripts/report-data/embedded-parity-baseline.json', 'utf8') | |
| ); | |
| const failures = styles.filter((style) => style.error || style.qualityBreakdown?.error); | |
| for (const style of styles) { | |
| console.log(`${style.name}: fidelity=${style.fidelityScore}`); | |
| } | |
| for (const style of failures) { | |
| console.error( | |
| `Selected fidelity failed for ${style.name}: ${style.error || style.qualityBreakdown?.error}` | |
| ); | |
| } | |
| // Styles already counted in `failures` (a measurement error, e.g. a | |
| // stale/missing citeproc snapshot) have untrustworthy fidelity and | |
| // exact-parity numbers this run — skip them here so a measurement | |
| // gap doesn't also get reported as a false fidelity/parity | |
| // regression; `failures.length > 0` below still fails the build. | |
| const unmeasurableNames = new Set(failures.map((style) => style.name)); | |
| const fidelityGateFailures = []; | |
| for (const name of Object.keys(fidelityBaseline.styles || {})) { | |
| const style = styleMap.get(name); | |
| if (!style || unmeasurableNames.has(name)) continue; | |
| if (Number(style.fidelityScore) < 1.0) { | |
| fidelityGateFailures.push(`${name}: fidelity=${style.fidelityScore} (must be 1.0)`); | |
| } | |
| } | |
| const parityGateFailures = []; | |
| for (const [name, baselineStyle] of Object.entries(parityBaseline.styles || {})) { | |
| const style = styleMap.get(name); | |
| if (!style || unmeasurableNames.has(name)) continue; | |
| const baselineParity = baselineStyle.exactParity; | |
| const currentParity = style.exactParity; | |
| if (!baselineParity) continue; | |
| if (Number(currentParity?.total) !== Number(baselineParity.total)) { | |
| parityGateFailures.push( | |
| `${name}: exact-parity total ${currentParity?.total} != baseline total ${baselineParity.total} ` + | |
| `(regenerate scripts/report-data/embedded-parity-baseline.json if intended)` | |
| ); | |
| } else if (Number(currentParity?.passed) < Number(baselineParity.passed)) { | |
| parityGateFailures.push( | |
| `${name}: exact-parity passed=${currentParity?.passed} < baseline floor ${baselineParity.passed}` | |
| ); | |
| } | |
| } | |
| if (fidelityGateFailures.length > 0 || parityGateFailures.length > 0) { | |
| for (const msg of fidelityGateFailures) console.error(`Fidelity gate failed for ${msg}`); | |
| for (const msg of parityGateFailures) console.error(`Exact-parity gate failed for ${msg}`); | |
| process.exit(1); | |
| } | |
| if (failures.length > 0) { | |
| process.exit(1); | |
| } | |
| NODE | |
| else | |
| node scripts/report-core.js --all-features > /tmp/core-report.json | |
| node scripts/check-core-quality.js \ | |
| --report /tmp/core-report.json \ | |
| --baseline scripts/report-data/core-quality-baseline.json \ | |
| --parity-baseline scripts/report-data/embedded-parity-baseline.json \ | |
| --parity-adjudication scripts/report-data/parity-adjudication.json | |
| fi | |
| - name: Check oracle top-10 regression baseline | |
| if: ${{ steps.fidelity-scope.outputs.mode == 'all' }} | |
| run: | | |
| node scripts/check-oracle-regression.js \ | |
| --baseline scripts/report-data/oracle-top10-baseline.json | |
| - name: Display sccache stats | |
| run: sccache --show-stats |