perf: shared generation prep job for parity pipeline

Add a 'generate' job that runs TYPECHECK_ONLY once for all 28 packages,
caches the output, then firely/hl7/internal jobs restore that cache and
set SKIP_GENERATION=1 to skip Phase 1 entirely.

Before: each of 57+ jobs (28 firely + 28 hl7 + 1 internal) downloaded
and generated all packages independently.
After: 1 prep job generates, 57 jobs validate only.

Also removed redundant cache steps (fhir-resources, seed-fhir-cache)
from validator jobs since generated output is fully cached.

Author: quotentiroler

.github/workflows/_parity-tests.yml

@@ -64,6 +64,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
+      cache-key: ${{ steps.cache-key.outputs.key }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -75,10 +76,86 @@ jobs:
           PACKAGES=$(node scripts/list-parity-packages.js "${{ inputs.packages }}")
           echo "matrix=$PACKAGES" >> "$GITHUB_OUTPUT"
           echo "Packages to test: $PACKAGES"
+      - name: Compute shared cache key
+        id: cache-key
+        run: |
+          KEY="fhir-generated-${{ hashFiles('src/test/pipelineParity.test.ts', 'src/generator/**/*.ts', 'src/generator/fhir/**/*.json') }}-v1"
+          echo "key=$KEY" >> "$GITHUB_OUTPUT"
+
+  # ── Generate all packages once (shared prep job) ───────────────────────────
+  generate:
+    needs: setup
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci || (npm cache clean --force && npm ci)
+
+      - name: Restore FHIR packages cache
+        id: cache-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: .cache/.fhir/packages
+          key: ${{ needs.setup.outputs.cache-key }}
+
+      - name: Cache tx-server ValueSet expansions
+        uses: actions/cache@v4
+        with:
+          path: .cache/tx-expansions
+          key: tx-expansions-v2
+          restore-keys: tx-expansions-v2
+
+      - name: Cache FHIR resources (StructureDefinitions)
+        uses: actions/cache@v4
+        with:
+          path: .cache/fhir-resources
+          key: fhir-resources-v4
+          restore-keys: fhir-resources-v4
+
+      - name: Seed FHIR resources cache
+        run: bash scripts/seed-fhir-cache.sh
+
+      - name: Build zod-r4 runtime package
+        run: |
+          cd packages/zod-r4
+          npm ci || (npm cache clean --force && npm ci)
+          npm run build
+
+      - name: Clear generated code to force regeneration
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        run: |
+          if [ -d ".cache/.fhir/packages" ]; then
+            find .cache/.fhir/packages -type d -name "generated" -exec rm -rf {} + 2>/dev/null || true
+          fi
+
+      - name: Generate all packages
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        run: npm run test:parity
+        env:
+          CI: true
+          PIPELINE_PACKAGES: ${{ inputs.packages }}
+          TYPECHECK_ONLY: '1'
+
+      - name: Save FHIR packages cache (with generated output)
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: .cache/.fhir/packages
+          key: ${{ needs.setup.outputs.cache-key }}
 
   # ── Firely (.NET) — one job per IG ─────────────────────────────────────────
   firely:
-    needs: [setup, resolve-versions]
+    needs: [setup, resolve-versions, generate]
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -118,11 +195,11 @@ jobs:
           rm -rf /tmp/FhirArtifactCache-* 2>/dev/null || true
           rm -rf /tmp/fhir-spec-cache-* 2>/dev/null || true
 
-      - name: Cache FHIR packages
-        uses: actions/cache@v4
+      - name: Restore pre-generated FHIR packages
+        uses: actions/cache/restore@v4
         with:
           path: .cache/.fhir/packages
-          key: fhir-packages-${{ matrix.package }}-${{ hashFiles('src/test/pipelineParity.test.ts', 'src/generator/**/*.ts', 'src/generator/fhir/**/*.json') }}-v8
+          key: ${{ needs.setup.outputs.cache-key }}
 
       - name: Cache tx-server ValueSet expansions
         uses: actions/cache@v4
@@ -131,33 +208,18 @@ jobs:
           key: tx-expansions-v2
           restore-keys: tx-expansions-v2
 
-      - name: Cache FHIR resources (StructureDefinitions)
-        uses: actions/cache@v4
-        with:
-          path: .cache/fhir-resources
-          key: fhir-resources-v4
-          restore-keys: fhir-resources-v4
-
-      - name: Seed FHIR resources cache
-        run: bash scripts/seed-fhir-cache.sh
-
       - name: Build zod-r4 runtime package
         run: |
           cd packages/zod-r4
           npm ci || (npm cache clean --force && npm ci)
           npm run build
 
-      - name: Clear generated code to force regeneration
-        run: |
-          if [ -d ".cache/.fhir/packages" ]; then
-            find .cache/.fhir/packages -type d -name "generated" -exec rm -rf {} + 2>/dev/null || true
-          fi
-
       - name: Run Firely parity test (${{ matrix.package }})
         id: firely-test
         run: npm run test:parity
         env:
           CI: true
+          SKIP_GENERATION: '1'
           PIPELINE_PACKAGES: ${{ matrix.package }}
           FIRELY_SDK_VERSION: ${{ needs.resolve-versions.outputs.firely_version }}
         continue-on-error: true
@@ -224,7 +286,7 @@ jobs:
 
   # ── HL7 Java Validator — one job per IG ────────────────────────────────────
   hl7:
-    needs: [setup, resolve-versions]
+    needs: [setup, resolve-versions, generate]
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -264,11 +326,11 @@ jobs:
           VALIDATOR_VERSION=${{ needs.resolve-versions.outputs.hl7_version }} \
             scripts/hl7-validator/validate.sh --download-only
 
-      - name: Cache FHIR packages
-        uses: actions/cache@v4
+      - name: Restore pre-generated FHIR packages
+        uses: actions/cache/restore@v4
         with:
           path: .cache/.fhir/packages
-          key: fhir-packages-${{ matrix.package }}-${{ hashFiles('src/test/pipelineParity.test.ts', 'src/generator/**/*.ts', 'src/generator/fhir/**/*.json') }}-v8
+          key: ${{ needs.setup.outputs.cache-key }}
 
       - name: Cache tx-server ValueSet expansions
         uses: actions/cache@v4
@@ -277,33 +339,18 @@ jobs:
           key: tx-expansions-v2
           restore-keys: tx-expansions-v2
 
-      - name: Cache FHIR resources (StructureDefinitions)
-        uses: actions/cache@v4
-        with:
-          path: .cache/fhir-resources
-          key: fhir-resources-v4
-          restore-keys: fhir-resources-v4
-
-      - name: Seed FHIR resources cache
-        run: bash scripts/seed-fhir-cache.sh
-
       - name: Build zod-r4 runtime package
         run: |
           cd packages/zod-r4
           npm ci || (npm cache clean --force && npm ci)
           npm run build
 
-      - name: Clear generated code to force regeneration
-        run: |
-          if [ -d ".cache/.fhir/packages" ]; then
-            find .cache/.fhir/packages -type d -name "generated" -exec rm -rf {} + 2>/dev/null || true
-          fi
-
       - name: Run HL7 parity test (${{ matrix.package }})
         id: hl7-test
         run: npm run test:parity
         env:
           CI: true
+          SKIP_GENERATION: '1'
           PIPELINE_PACKAGES: ${{ matrix.package }}
           FHIR_VALIDATOR: hl7
           HL7_VALIDATOR_VERSION: ${{ needs.resolve-versions.outputs.hl7_version }}
@@ -371,6 +418,7 @@ jobs:
 
   # ── Internal (validate) — single job, all IGs sequentially ──────────
   internal:
+    needs: [setup, generate]
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -388,11 +436,11 @@ jobs:
       - name: Install dependencies
         run: npm ci || (npm cache clean --force && npm ci)
 
-      - name: Cache FHIR packages
-        uses: actions/cache@v4
+      - name: Restore pre-generated FHIR packages
+        uses: actions/cache/restore@v4
         with:
           path: .cache/.fhir/packages
-          key: fhir-packages-all-${{ hashFiles('src/test/pipelineParity.test.ts', 'src/generator/**/*.ts', 'src/generator/fhir/**/*.json') }}-v8
+          key: ${{ needs.setup.outputs.cache-key }}
 
       - name: Cache tx-server ValueSet expansions
         uses: actions/cache@v4
@@ -401,33 +449,18 @@ jobs:
           key: tx-expansions-v2
           restore-keys: tx-expansions-v2
 
-      - name: Cache FHIR resources (StructureDefinitions)
-        uses: actions/cache@v4
-        with:
-          path: .cache/fhir-resources
-          key: fhir-resources-v4
-          restore-keys: fhir-resources-v4
-
-      - name: Seed FHIR resources cache
-        run: bash scripts/seed-fhir-cache.sh
-
       - name: Build zod-r4 runtime package
         run: |
           cd packages/zod-r4
           npm ci || (npm cache clean --force && npm ci)
           npm run build
 
-      - name: Clear generated code to force regeneration
-        run: |
-          if [ -d ".cache/.fhir/packages" ]; then
-            find .cache/.fhir/packages -type d -name "generated" -exec rm -rf {} + 2>/dev/null || true
-          fi
-
       - name: Run internal parity tests (all packages)
         id: internal-test
         run: npm run test:parity
         env:
           CI: true
+          SKIP_GENERATION: '1'
           PIPELINE_PACKAGES: ${{ inputs.packages }}
           FHIR_VALIDATOR: internal
         continue-on-error: true

src/test/pipelineParity.test.ts

@@ -34,6 +34,7 @@ import {
  * - FHIR_VALIDATOR: Which validator to use: 'firely' (default) or 'hl7'
  * - HL7_VALIDATOR_VERSION: HL7 validator version (default: 6.9.4)
  * - TYPECHECK_ONLY: Only run generation + TypeScript compilation check (no validator, no resource gen)
+ * - SKIP_GENERATION: Reuse pre-generated output from FHIR package cache (set by CI prep job)
  * - ALLOW_GENERATION_FAILURES: Continue test even if generation fails (report errors without aborting)
  * 
  * Generation always uses --tx-server https://tx.fhir.org/r4 to expand ValueSets
@@ -61,6 +62,7 @@ const SKIP_CLEANUP = process.env.SKIP_CLEANUP === '1' || process.env.SKIP_CLEANU
 const VERBOSE = process.env.VERBOSE_TESTS === '1' || process.env.VERBOSE_TESTS === 'true';
 const TYPECHECK_ONLY = process.env.TYPECHECK_ONLY === '1' || process.env.TYPECHECK_ONLY === 'true';
 const ALLOW_GENERATION_FAILURES = process.env.ALLOW_GENERATION_FAILURES === '1' || process.env.ALLOW_GENERATION_FAILURES === 'true';
+const SKIP_GENERATION = process.env.SKIP_GENERATION === '1' || process.env.SKIP_GENERATION === 'true';
 const FHIR_RELEASE = process.env.FHIR_RELEASE || 'R4';
 
 // Filter to only requested packages
@@ -190,7 +192,7 @@ ${rows}
 
     console.log(`\n${'═'.repeat(70)}`);
     console.log(`📦 Pipeline Parity — ${selectedPackages.length} package(s) SEQUENTIALLY`);
-    console.log(`   Validator: ${VALIDATOR_TYPE.toUpperCase()} | FHIR: ${FHIR_RELEASE} | TYPECHECK_ONLY: ${TYPECHECK_ONLY}`);
+    console.log(`   Validator: ${VALIDATOR_TYPE.toUpperCase()} | FHIR: ${FHIR_RELEASE} | TYPECHECK_ONLY: ${TYPECHECK_ONLY}${SKIP_GENERATION ? ' | SKIP_GENERATION: true' : ''}`);
     console.log(`   Packages: ${selectedPackages.map(p => p.name).join(', ')}`);
     console.log(`   Heap: ${formatMB(process.memoryUsage().heapUsed)} (initial)`);
     console.log(`${'═'.repeat(70)}`);
@@ -207,23 +209,50 @@ ${rows}
     // PHASE 1: Generate TypeScript for all packages SEQUENTIALLY to avoid cache conflicts
     // Each package generation fetches base FHIR StructureDefinitions from hl7.org and caches them
     // Parallel runs can corrupt the cache, causing some packages to fail with TypeScript errors
-    console.log(`\n[${timestamp()}] 🚀 Phase 1: Downloading and generating ${selectedPackages.length} packages...`);
+    //
+    // SKIP_GENERATION: When set, reuse pre-generated output from the FHIR package cache.
+    // A dedicated CI prep job generates all packages once; validator jobs skip regeneration.
     const generationResults: Array<PromiseSettledResult<{ pkg: PackageConfig; tgzPath: string; packageDir: string; classInfos: ClassInfo[] }>> = [];
-    
-    for (let genIdx = 0; genIdx < selectedPackages.length; genIdx++) {
-      const pkg = selectedPackages[genIdx];
-      const genStart = Date.now();
-      console.log(`[${timestamp()}]   [${genIdx + 1}/${selectedPackages.length}] Generating ${pkg.name} (${pkg.spec})...`);
-      try {
-        const { tgzPath, packageDir } = await generateForPackage(pkg, { repoRoot, validatorType: VALIDATOR_TYPE, verbose: VERBOSE });
+
+    if (SKIP_GENERATION) {
+      console.log(`\n[${timestamp()}] ⏭ Phase 1: SKIP_GENERATION — reusing pre-generated output for ${selectedPackages.length} packages...`);
+      const { getFhirPackagesCacheDir } = await import('../generator/core/cacheConfig.js');
+      const fhirPackageCacheDir = getFhirPackagesCacheDir();
+
+      for (let genIdx = 0; genIdx < selectedPackages.length; genIdx++) {
+        const pkg = selectedPackages[genIdx];
+        const [pkgName, version] = pkg.spec.split('@');
+        const packageDir = path.join(fhirPackageCacheDir, `${pkgName}@${version}`);
+        const generatedDir = path.join(packageDir, 'generated');
+
+        if (!fs.existsSync(generatedDir)) {
+          console.error(`[${timestamp()}]   [${genIdx + 1}/${selectedPackages.length}] ✗ ${pkg.name}: generated/ not found at ${generatedDir}`);
+          generationResults.push({ status: 'rejected', reason: new Error(`Pre-generated output not found for ${pkg.name}: ${generatedDir}`) });
+          continue;
+        }
+
         const classInfos = listGeneratedClassFiles(packageDir, pkg.name);
-        const genElapsed = ((Date.now() - genStart) / 1000).toFixed(1);
-        console.log(`[${timestamp()}]   [${genIdx + 1}/${selectedPackages.length}] ✓ ${pkg.name}: ${classInfos.length} classes (${genElapsed}s)`);
-        generationResults.push({ status: 'fulfilled', value: { pkg, tgzPath, packageDir, classInfos } });
-      } catch (error) {
-        const genElapsed = ((Date.now() - genStart) / 1000).toFixed(1);
-        console.error(`[${timestamp()}]   [${genIdx + 1}/${selectedPackages.length}] ✗ ${pkg.name} FAILED (${genElapsed}s):`, error);
-        generationResults.push({ status: 'rejected', reason: error });
+        console.log(`[${timestamp()}]   [${genIdx + 1}/${selectedPackages.length}] ✓ ${pkg.name}: ${classInfos.length} classes (cached)`);
+        generationResults.push({ status: 'fulfilled', value: { pkg, tgzPath: '', packageDir, classInfos } });
+      }
+    } else {
+      console.log(`\n[${timestamp()}] 🚀 Phase 1: Downloading and generating ${selectedPackages.length} packages...`);
+    
+      for (let genIdx = 0; genIdx < selectedPackages.length; genIdx++) {
+        const pkg = selectedPackages[genIdx];
+        const genStart = Date.now();
+        console.log(`[${timestamp()}]   [${genIdx + 1}/${selectedPackages.length}] Generating ${pkg.name} (${pkg.spec})...`);
+        try {
+          const { tgzPath, packageDir } = await generateForPackage(pkg, { repoRoot, validatorType: VALIDATOR_TYPE, verbose: VERBOSE });
+          const classInfos = listGeneratedClassFiles(packageDir, pkg.name);
+          const genElapsed = ((Date.now() - genStart) / 1000).toFixed(1);
+          console.log(`[${timestamp()}]   [${genIdx + 1}/${selectedPackages.length}] ✓ ${pkg.name}: ${classInfos.length} classes (${genElapsed}s)`);
+          generationResults.push({ status: 'fulfilled', value: { pkg, tgzPath, packageDir, classInfos } });
+        } catch (error) {
+          const genElapsed = ((Date.now() - genStart) / 1000).toFixed(1);
+          console.error(`[${timestamp()}]   [${genIdx + 1}/${selectedPackages.length}] ✗ ${pkg.name} FAILED (${genElapsed}s):`, error);
+          generationResults.push({ status: 'rejected', reason: error });
+        }
       }
     }