fix(ci): scan the pinned upstream images before the release, not at it

.github/workflows/ci-security-scan.yml

@@ -8,10 +8,57 @@ on:
         required: false
         type: string
         default: "false"
+      release_images_changed:
+        description: "Whether to scan the upstream images pinned in security/release-images.json"
+        required: false
+        type: string
+        default: "false"
 
 permissions: {}
 
 jobs:
+  upstream-images:
+    name: "Pinned upstream images"
+    runs-on: ubuntu-latest
+    # The four upstream images are shipped by digest and never built here, so no build gate has a
+    # subject for them and the release evidence gate was the first thing that scanned them — which
+    # is how v0.75.0 discovered a days-old finding at the worst possible moment (#1741). A digest
+    # needs no build, so the pull request that changes one scans it.
+    if: inputs.should_skip != 'true' && inputs.release_images_changed == 'true'
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      - uses: ./.github/actions/setup-node-pnpm
+        with:
+          install: "none"
+
+      - uses: ./.github/actions/setup-release-security-tools
+        with:
+          install-syft: "false"
+
+      # A blocking gate inherits Trivy's database availability as a hard dependency, so the
+      # retries and the mirror fallback live in one action rather than in each caller.
+      - uses: ./.github/actions/download-trivy-db
+
+      - name: Enforce the release vulnerability policy on the pinned digests
+        run: node scripts/scan-upstream-images.ts reports
+
+      # The gate's exit status alone cannot say which digest carries which finding.
+      - name: Upload the vulnerability policy results
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: vulnerability-policy-upstream
+          path: reports
+          if-no-files-found: warn
+          retention-days: 7
+
   dependency-review:
     name: "New dependency risk"
     runs-on: ubuntu-latest

.github/workflows/cicd.yml

@@ -36,6 +36,7 @@ jobs:
       build-config: ${{ steps.filter.outputs.build-config }}
       test-config: ${{ steps.filter.outputs.test-config }}
       security-config: ${{ steps.filter.outputs.security-config }}
+      release-images: ${{ steps.filter.outputs.release-images }}
       docker-config: ${{ steps.filter.outputs.docker-config }}
       pmd-canary: ${{ steps.filter.outputs.pmd-canary }}
       version-bump: ${{ steps.version_bump.outputs.changed }}
@@ -196,6 +197,19 @@ jobs:
             security-config:
               - '.github/workflows/cicd.yml'
               - '.github/workflows/ci-security-scan.yml'
+            release-images:
+              # The pinned upstream digests need no build, so the pull request that edits them —
+              # a Renovate digest bump included — is the one that scans them (#1741). The script
+              # entries are the gate's whole transitive import closure, which ci-contract.test.ts
+              # re-derives from the imports and asserts against this list.
+              - 'security/release-images.json'
+              - 'security/vulnerability-policy.json'
+              - 'scripts/scan-upstream-images.ts'
+              - 'scripts/check-release-vulnerabilities.ts'
+              - 'scripts/lib/image-scan.ts'
+              - 'scripts/lib/json.ts'
+              - 'scripts/lib/process.ts'
+              - '.github/workflows/ci-security-scan.yml'
             docker-config:
               - '.github/workflows/ci-docker-build.yml'
               - '.github/workflows/reusable-docker-build.yml'
@@ -329,13 +343,15 @@ jobs:
       needs.detect-changes.outputs.should_skip != 'true' && (
         needs.detect-changes.outputs.any-code == 'true' ||
         needs.detect-changes.outputs.security-config == 'true' ||
+        needs.detect-changes.outputs.release-images == 'true' ||
         github.event_name != 'pull_request'
       )
     permissions:
       contents: read
       security-events: write
     with:
       should_skip: ${{ needs.detect-changes.outputs.should_skip }}
+      release_images_changed: ${{ (needs.detect-changes.outputs.release-images == 'true' || github.event_name != 'pull_request') && 'true' || 'false' }}
 
   Test:
     uses: ./.github/workflows/ci-tests.yml

.github/workflows/rescan-main-images.yml

@@ -55,6 +55,12 @@ jobs:
           TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
         run: node scripts/scan-main-images.ts reports
 
+      # The pinned upstream digests too, so the weekly picture is the release gate's subject set
+      # rather than half of it. Report-only for the same reason as the images above: a CVE
+      # published after the digest was pinned belongs to no commit here.
+      - name: Scan the pinned upstream images
+        run: node scripts/scan-upstream-images.ts reports --report-only
+
       - name: Upload the vulnerability policy results
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

docs/contributor/vulnerability-remediation.mdx

@@ -29,15 +29,33 @@ attached to the release; the filter is in the evaluator, not a `--ignore-unfixed
 | When | Subject | Effect |
 | --- | --- | --- |
 | Every image build (`reusable-docker-build.yml`) | the `linux/amd64` image the run just pushed | fails the run |
+| A pull request touching `security/release-images.json` (`ci-security-scan.yml`) | both platforms of each pinned upstream digest | fails the run |
 | Release evidence (`release.yml`) | both platforms of every first-party and upstream image | no release is created |
-| Weekly (`rescan-main-images.yml`) | the `linux/amd64` image behind each `:main` tag | opens or updates a tracking issue |
+| Weekly (`rescan-main-images.yml`) | the `linux/amd64` image behind each `:main` tag, and both platforms of each pinned upstream digest | opens or updates a tracking issue |
 | Weekly (`rescan-release-images.yml`) | every subject in the latest published release | fails and comments on [the vulnerability response issue](https://github.com/ls1intum/Hephaestus/issues/1369) |
 
-All four evaluate `security/vulnerability-policy.json` through `scripts/check-release-vulnerabilities.ts`;
-the release and the `main` rescan reach it through `verify-release-evidence.ts` and
-`scan-main-images.ts`, and `scripts/ci-contract.test.ts` asserts there is one policy file. The
-scheduled rescans do not fail a commit status: a finding published after a clean merge has no commit
-to revert. A scanner failure is never treated as no findings.
+All five evaluate `security/vulnerability-policy.json` through `scripts/check-release-vulnerabilities.ts`;
+the release and the rescans reach it through `verify-release-evidence.ts`, `scan-main-images.ts` and
+`scan-upstream-images.ts`, and `scripts/ci-contract.test.ts` asserts there is one policy file. That
+test also asserts the scans before the release cover exactly the subject set the release gate covers,
+both derived from `security/release-images.json`: the upstream images are shipped by digest and never
+built here, so nothing scanned them until the release gate did, and v0.75.0 failed on a finding that
+had been in the pinned alpine digest for days. The scheduled rescans do not fail a commit status: a
+finding published after a clean merge has no commit to revert. A scanner failure is never treated as
+no findings.
+
+A pinned upstream image cannot be patched by rebuilding anything here. Its remedy is a digest bump in
+`security/release-images.json` — which Renovate proposes, and which the pull request proposing it now
+scans — and until upstream republishes the tag there may be no digest to bump to.
+
+That is also why the upstream scan covers both platforms while the scans of images we build cover
+`linux/amd64` only. An exception matches on `image | platform | vulnerability | package |
+installedVersion`, so a single-platform scan can only ever half-check a subject. For an image we
+build, the other half costs little to leave to the release: a finding on either architecture is
+fixed by the same rebuild, and the Version PR preflight scans both before the release is cut. For a
+pinned upstream digest there is no rebuild, the fix is a digest bump nobody can make until upstream
+publishes one, and the bump arrives in a pull request — so an arm64-only finding has to fail that
+pull request rather than the release.
 
 ## Exceptions
 

scripts/ci-contract.test.ts

@@ -1,10 +1,15 @@
 import assert from "node:assert/strict";
+import { existsSync } from "node:fs";
 import { glob, readFile } from "node:fs/promises";
+import path from "node:path";
 import { describe, test } from "node:test";
 
 import { type Document, isMap, isSeq, parseDocument, type YAMLMap } from "yaml";
 
 import { planRelease, releaseOutputs } from "./plan-release.ts";
+import { planSubjects } from "./scan-main-images.ts";
+import { PLATFORMS, planUpstreamSubjects } from "./scan-upstream-images.ts";
+import { validateManifest } from "./verify-release-evidence.ts";
 
 function job(source: string, name: string): string {
 	const match = source.match(
@@ -37,6 +42,30 @@ function escapeRegExp(value: string): string {
 	return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
 
+/**
+ * Every repository script a given entry point loads, transitively — its static relative imports,
+ * plus any sibling script it names as a string, which is how the scanners reach the policy
+ * evaluator (they spawn it rather than importing it, so that its exit status is the verdict).
+ */
+async function importClosure(entry: string): Promise<string[]> {
+	const seen = new Set<string>();
+	const queue = [entry];
+	while (queue.length > 0) {
+		const file = queue.shift();
+		if (file === undefined || seen.has(file)) continue;
+		seen.add(file);
+		const source = await readFile(file, "utf8");
+		const directory = path.dirname(file);
+		for (const [, specifier] of source.matchAll(/from\s+"(\.[^"]+\.ts)"/g))
+			queue.push(path.normalize(path.join(directory, specifier ?? "")));
+		for (const [, name] of source.matchAll(/"([\w.-]+\.ts)"/g)) {
+			const candidate = path.normalize(path.join(directory, "..", name ?? ""));
+			if (candidate.startsWith("scripts/") && existsSync(candidate)) queue.push(candidate);
+		}
+	}
+	return [...seen].toSorted();
+}
+
 /** The `with` map of the first step in a job whose `uses` starts with `action`. */
 function step(workflow: Document, jobPath: string[], action: string): YAMLMap {
 	const steps = workflow.getIn([...jobPath, "steps"]);
@@ -392,6 +421,80 @@ void describe("CI contract", () => {
 		]);
 	});
 
+	void test("scans every release subject before the release, not only at the release gate", async () => {
+		const inventory: unknown = JSON.parse(await readFile("security/release-images.json", "utf8"));
+		const namespace = "ghcr.io/hephaestus-build";
+		const digest = `sha256:${"c".repeat(64)}`;
+		// The subject set the pre-release scans cover, derived from the inventory rather than listed:
+		// the build gate and the weekly rescan take the first-party half, scan-upstream-images.ts the
+		// pinned upstream half.
+		const scanned = [
+			...planSubjects(inventory, namespace, "main").map((subject) => ({
+				...subject,
+				indexDigest: digest,
+				provenance: "first-party" as const,
+			})),
+			...planUpstreamSubjects(inventory).map((subject) => ({
+				...subject,
+				provenance: "upstream" as const,
+			})),
+		];
+		// Parity with the release gate, asserted by the release gate itself: this is the manifest that
+		// would evidence exactly the pre-release subject set, and validateManifest rejects a manifest
+		// whose subjects are not exactly the inventory. So an image the pre-release scans miss, or one
+		// they cover that the release does not, fails here — which is what v0.75.0 needed and did not
+		// have when the upstream half was scanned nowhere before the release (#1741).
+		const manifest = {
+			schemaVersion: 1,
+			subjects: scanned.flatMap((subject) =>
+				(["linux/amd64", "linux/arm64"] as const).map((platform) => ({
+					digest,
+					image: subject.image,
+					indexDigest: subject.indexDigest,
+					platform,
+					provenance: subject.provenance,
+					repository: subject.repository,
+				})),
+			),
+		};
+		assert.doesNotThrow(() => validateManifest(manifest, inventory, namespace));
+
+		// A planner nothing invokes covers nothing. The pinned digests need no build, so they are
+		// scanned on the pull request that changes them — which is the pull request a Renovate digest
+		// bump opens — and again in the weekly rescan, where a finding routes to the tracking issue.
+		assert.match(
+			await readFile(".github/workflows/ci-security-scan.yml", "utf8"),
+			/run: node scripts\/scan-upstream-images\.ts reports\n/,
+		);
+		assert.match(
+			await readFile(".github/workflows/rescan-main-images.yml", "utf8"),
+			/run: node scripts\/scan-upstream-images\.ts reports --report-only\n/,
+		);
+		const detection = job(await readFile(".github/workflows/cicd.yml", "utf8"), "detect-changes");
+		const filter = pathFilter(detection, "release-images");
+		assert.match(
+			filter,
+			/- 'security\/release-images\.json'[\s\S]*- 'security\/vulnerability-policy\.json'/,
+		);
+		// The trigger is derived, not trusted: a filter that lists the entry point but not the module
+		// it parses JSON with skips the gate on the pull request that breaks the parser. Re-walk the
+		// imports and require every file the gate actually loads to appear.
+		for (const file of await importClosure("scripts/scan-upstream-images.ts"))
+			assert.ok(
+				filter.includes(`- '${file}'`),
+				`release-images must trigger on ${file}, which the upstream scan loads`,
+			);
+	});
+
+	void test("scans both released platforms before the release, not just linux/amd64", async () => {
+		// The policy match key is `image | platform | vulnerability | package | installedVersion`, so
+		// a single-platform pre-release scan leaves an arm64-only finding — or an arm64 exception
+		// nobody wrote — to be discovered by the release gate, which is the failure this PR removes.
+		assert.deepEqual([...PLATFORMS], ["linux/amd64", "linux/arm64"]);
+		const source = await readFile("scripts/scan-upstream-images.ts", "utf8");
+		assert.match(source, /for \(const platform of PLATFORMS\)/);
+	});
+
 	void test("rescans main's images weekly and reports drift to an issue, not a status", async () => {
 		const source = await readFile(".github/workflows/rescan-main-images.yml", "utf8");
 		// Weekly, matching the supported-release rescan: the remedy for drift is a rebuild of main,