Keep candidate maintenance from moving release tags

Author: dhawal-ss

.github/workflows/container-candidates.yml

@@ -2,6 +2,12 @@ name: Mesh container candidates
 
 on:
   workflow_dispatch:
+    inputs:
+      publish:
+        description: Publish a new, precomputed candidate digest under a release tag that does not exist yet
+        required: true
+        type: boolean
+        default: false
   push:
     branches: [main, beta/production-readiness-2026-08]
     paths:
@@ -28,7 +34,7 @@ permissions:
 
 jobs:
   publish:
-    name: Publish and verify Mesh container candidate (${{ matrix.name }})
+    name: Maintain and verify Mesh container candidate (${{ matrix.name }})
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -37,11 +43,13 @@ jobs:
           - name: caddy
             image: ghcr.io/dhawal-ss/mesh-caddy
             tag: 2.11.4-mesh.1
+            digest: sha256:67cf933778f84aadd59f93fc5d2eaf6471df7b415135ae95fc1b1e6e5f4b7e4f
             context: mesh/infra/container-candidates/caddy
             dockerfile: mesh/infra/container-candidates/caddy/Dockerfile
           - name: lk-jwt-service
             image: ghcr.io/dhawal-ss/mesh-lk-jwt-service
             tag: 0.5.0-mesh.1
+            digest: sha256:78bf2f1e8535928037abc35a9886614b3f722885b54f1f707a570e6991252710
             context: mesh/infra/container-candidates/lk-jwt-service
             dockerfile: mesh/infra/container-candidates/lk-jwt-service/Dockerfile
     steps:
@@ -54,41 +62,65 @@ jobs:
         run: npm run check:container-candidates
       - name: Set up Buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+      - name: Refuse an existing release tag before publication
+        if: github.event_name == 'workflow_dispatch' && inputs.publish
+        shell: bash
+        run: |
+          if docker buildx imagetools inspect '${{ matrix.image }}:${{ matrix.tag }}' >/dev/null 2>&1; then
+            echo 'Candidate release tags are immutable. Choose a new reviewed tag.' >&2
+            exit 1
+          fi
       - name: Sign in to GitHub Container Registry
+        if: github.event_name == 'workflow_dispatch' && inputs.publish
         uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build and publish exact multi-platform candidate
+      - name: Build and publish staged multi-platform candidate
+        if: github.event_name == 'workflow_dispatch' && inputs.publish
         id: build
         uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
         with:
           context: ${{ matrix.context }}
           file: ${{ matrix.dockerfile }}
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ${{ matrix.image }}:${{ matrix.tag }}
-          provenance: mode=max
+          tags: ${{ matrix.image }}:staging-${{ github.run_id }}-${{ github.run_attempt }}
+          provenance: false
           sbom: false
           cache-from: type=gha,scope=${{ matrix.name }}
           cache-to: type=gha,mode=max,scope=${{ matrix.name }}
       - name: Resolve exact candidate image
         id: exact
         shell: bash
         run: |
-          exact_image='${{ matrix.image }}:${{ matrix.tag }}@${{ steps.build.outputs.digest }}'
+          expected_digest='${{ matrix.digest }}'
+          built_digest='${{ steps.build.outputs.digest }}'
+          if [ -n "$built_digest" ]; then
+            if [ "$built_digest" != "$expected_digest" ]; then
+              echo "The staged build digest does not match the precomputed reviewed digest." >&2
+              exit 1
+            fi
+            docker buildx imagetools create \
+              --tag '${{ matrix.image }}:${{ matrix.tag }}' \
+              '${{ matrix.image }}@${{ matrix.digest }}'
+          fi
+          exact_image='${{ matrix.image }}:${{ matrix.tag }}@${{ matrix.digest }}'
+          docker buildx imagetools inspect "$exact_image" >/dev/null
           echo "image=$exact_image" >> "$GITHUB_OUTPUT"
-          echo "Published \`$exact_image\`" >> "$GITHUB_STEP_SUMMARY"
+          echo "Verified \`$exact_image\`" >> "$GITHUB_STEP_SUMMARY"
       - name: Install pinned Cosign
         uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
         with:
           cosign-release: v3.0.6
       - name: Sign exact candidate digest with GitHub OIDC
+        if: github.event_name == 'workflow_dispatch' && inputs.publish
         env:
           EXACT_IMAGE: ${{ steps.exact.outputs.image }}
         run: cosign sign --yes "$EXACT_IMAGE"
       - name: Publish registry provenance attestation
+        if: github.event_name == 'workflow_dispatch' && inputs.publish
         uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2
         with:
           subject-name: ${{ matrix.image }}
@@ -121,6 +153,7 @@ jobs:
           --image '${{ steps.exact.outputs.image }}'
           --output "candidate-${{ matrix.name }}-scan-evidence.json"
       - name: Attach the pinned CycloneDX SBOM to the exact digest
+        if: github.event_name == 'workflow_dispatch' && inputs.publish
         env:
           EXACT_IMAGE: ${{ steps.exact.outputs.image }}
         run: >-

mesh/docs/operations/CONTAINER_CANDIDATE_MAINTENANCE.md

@@ -16,19 +16,25 @@ exception or unsupported dependency combination.
 ## Build and trust contract
 
 The workflow at `.github/workflows/container-candidates.yml` binds every build
-to an upstream release tag and commit. It builds Linux AMD64 and ARM64 images
-from a pinned builder, publishes only non-root scratch runtimes, emits a
-CycloneDX SBOM and registry provenance, rejects fixable High or Critical
-findings, and signs the exact multi-platform digest with GitHub Actions OIDC.
+to an upstream release tag and commit. An explicit manual publication builds
+Linux AMD64 and ARM64 images from a pinned builder, first pushes a run-specific
+staging tag, rejects any digest other than the precomputed reviewed digest, and
+refuses to overwrite an existing release tag. It publishes only non-root
+scratch runtimes, emits a CycloneDX SBOM and registry provenance, rejects
+fixable High or Critical findings, and signs the exact multi-platform digest
+with GitHub Actions OIDC.
 
 Candidate packages must be public. A private package is not eligible for a
 Compose or release-policy pin because community operators must not need a Mesh
 credential to retrieve an optional service image.
 
 ## Maintenance response
 
-- The protected workflow scans candidates every week and whenever an image,
-  policy, validation script, or workflow changes.
+- Pushes, scheduled maintenance, and ordinary manual runs only scan and verify
+  the policy-pinned exact digest. They do not build, push, sign, attest, or
+  change a registry tag.
+- Publication requires an explicit manual `publish` input, a new release tag,
+  and a precomputed digest. Release tags are immutable after that operation.
 - A fixable High or Critical finding blocks the candidate immediately. Begin a
   dependency update or rebuild within 24 hours.
 - Review an unfixed Critical finding within 24 hours. Do not add an exception

mesh/infra/container-candidates/candidate-policy.json

@@ -41,6 +41,7 @@
     {
       "name": "caddy",
       "image": "ghcr.io/dhawal-ss/mesh-caddy:2.11.4-mesh.1",
+      "publishedDigest": "sha256:67cf933778f84aadd59f93fc5d2eaf6471df7b415135ae95fc1b1e6e5f4b7e4f",
       "context": "infra/container-candidates/caddy",
       "dockerfile": "infra/container-candidates/caddy/Dockerfile",
       "sourceRepository": "https://github.com/caddyserver/caddy.git",
@@ -57,6 +58,7 @@
     {
       "name": "lk-jwt-service",
       "image": "ghcr.io/dhawal-ss/mesh-lk-jwt-service:0.5.0-mesh.1",
+      "publishedDigest": "sha256:78bf2f1e8535928037abc35a9886614b3f722885b54f1f707a570e6991252710",
       "context": "infra/container-candidates/lk-jwt-service",
       "dockerfile": "infra/container-candidates/lk-jwt-service/Dockerfile",
       "sourceRepository": "https://github.com/element-hq/lk-jwt-service.git",

mesh/reports/TRACK_A_REPORT.md

@@ -34,6 +34,7 @@ Mesh remains `NO_GO_PUBLIC_BETA`. Track A is not complete until the pull request
 | The release rung could not be fully green before Track A merge | Seven of nine release commands pass. `check:release-sboms` rejects the preceding text-build reachability file because it is not a Matrix voice artifact, and `release:preflight` stops on D7. | Record both failures. Do not run the Track E voice rung or mark protected gates from local output. |
 | Protected security evidence failed after the first push | Caddy, Synapse, and `lk-jwt-service` contained fixable High or Critical findings. Newer upstream-only tags still failed with 8, 32, and 3 High or Critical findings respectively. | Do not waive the findings. Maintain hardened Mesh candidates only for the two dependency graphs that pass upstream tests and the zero-finding scan. Keep Synapse blocked. |
 | The first exact-digest R3 run reached the new image but stopped before strict identity verification | Cosign `v3.0.6` returned one direct image signature together with the signed CycloneDX and SLSA referrers. The discovery normalizer correctly rejected the attestation payload types under its former direct-signature-only shape. | Keep unknown payloads rejected, require at least one direct image signature, separately count only the two reviewed attestation types, and retain the strict Mesh workflow-identity verification. |
+| A source-validator change retriggered candidate publication under the existing release tags | Cancellation stopped Caddy before publication, but the faster `lk-jwt-service` job had already moved the human-readable tag to a new same-source digest. The reviewed exact digest remained public, signed, zero-finding, and unchanged in every production reference. | Treat the digest as authoritative. Make push, schedule, and ordinary manual runs scan-only; require an explicit manual publish input, a precomputed digest, a staging tag, and refusal of any existing release tag. Do not use a human-readable tag as release trust. |
 
 ## Post-push container remediation
 
@@ -67,7 +68,7 @@ The owner approved a zero-cost public GHCR maintenance path on 2026-08-10. The i
 | 4, end to end | Pass, 3 of 3 commands | Chromium installation passed. Playwright: 87 of 87 cases. Production preview performance: interactive median 75 ms and worst observed 205 ms across three local samples. |
 | 5, release gates | Partial, 7 of 9 commands | Owner, beta, operations, readiness structure, protected-evidence verifier tests, installer payload, and coexistence passed. Release SBOM generation failed because the current `dist/mesh-reachability.json` is the text build and omits `livekit-client`. Preflight failed on D7. |
 | Additional commit contract | Pass | `cargo clippy --all-targets --locked -- -D warnings` passed with one build job. `git diff --check` passed. No secret-bearing filenames or high-confidence private-key and token patterns were found. |
-| Candidate container contract | Pass locally | 67 supply-chain tests pass. Both candidate Dockerfiles build from exact upstream commits. Caddy configuration and HTTP health pass; `lk-jwt-service` `/healthz` passes. Both run as `65532:65532` with read-only filesystems, dropped capabilities, and no new privileges. The complete local prototype run built both images reproducibly with zero fixable findings and reproduced the two blocked Synapse scans. Real Cosign output for both public digests normalizes to one direct signature and two reviewed attestations while rejecting attestation-only and unknown-type payloads. |
+| Candidate container contract | Pass locally | 68 supply-chain tests pass. Both candidate Dockerfiles build from exact upstream commits. Caddy configuration and HTTP health pass; `lk-jwt-service` `/healthz` passes. Both run as `65532:65532` with read-only filesystems, dropped capabilities, and no new privileges. The complete local prototype run built both images reproducibly with zero fixable findings and reproduced the two blocked Synapse scans. Real Cosign output for both public digests normalizes to one direct signature and two reviewed attestations while rejecting attestation-only and unknown-type payloads. Scheduled and push-triggered candidate maintenance is scan-only; publication is manual, precomputed-digest-bound, staged, and existing-tag-refusing. |
 | Candidate publication | Pass on pull request source | Workflow run `31377423123` built, published, signed, attested, scanned, verified, and retained evidence for both public multi-architecture images. Independent exact-digest verification accepted only the repository workflow identity and found zero fixable High or Critical findings on Linux AMD64 and ARM64. Anonymous manifest inspection passed for both packages. |
 | 6, voice specific | Not run | The work order limits this rung to Track E. Track A does not use it to claim voice readiness. |
 

mesh/scripts/check-container-candidates.mjs

@@ -6,6 +6,7 @@ import { fileURLToPath } from 'node:url'
 const SHA40 = /^[0-9a-f]{40}$/u
 const PINNED_BUILDER = /^[A-Za-z0-9./_-]+:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}$/u
 const CANDIDATE_IMAGE = /^ghcr\.io\/dhawal-ss\/[a-z0-9-]+:[A-Za-z0-9._-]+$/u
+const OCI_DIGEST = /^sha256:[0-9a-f]{64}$/u
 
 function hash(contents) {
   return createHash('sha256').update(contents).digest('hex')
@@ -31,6 +32,7 @@ export function validateContainerCandidates({ policy, workflow, dockerfiles, pat
   if (images.map((entry) => entry.name).join(',') !== 'caddy,lk-jwt-service') fail('candidate policy must contain exactly Caddy and lk-jwt-service')
   for (const image of images) {
     if (!CANDIDATE_IMAGE.test(image?.image ?? '')) fail(`${image?.name ?? 'candidate'} image tag is invalid`)
+    if (!OCI_DIGEST.test(image?.publishedDigest ?? '')) fail(`${image?.name ?? 'candidate'} published digest is invalid`)
     if (!SHA40.test(image?.sourceCommit ?? '')) fail(`${image?.name ?? 'candidate'} source commit is invalid`)
     if (!image?.sourceRelease?.startsWith('v')) fail(`${image?.name ?? 'candidate'} source release is invalid`)
     if (!image?.context?.startsWith(`infra/container-candidates/${image.name}`)
@@ -55,19 +57,26 @@ export function validateContainerCandidates({ policy, workflow, dockerfiles, pat
       if (!dockerfile.includes(path.posix.basename(patch.path)) || !dockerfile.includes(patch.sha256) || !dockerfile.includes('git apply --check')) fail(`${image.name} source patch is not checksum-bound and preflighted`)
     }
     const [repository, tag] = image.image.split(/:(?=[^/]+$)/u)
-    if (!workflow.includes(`image: ${repository}`) || !workflow.includes(`tag: ${tag}`)) fail(`${image.name} is missing from the publishing matrix`)
+    if (!workflow.includes(`image: ${repository}`)
+      || !workflow.includes(`tag: ${tag}`)
+      || !workflow.includes(`digest: ${image.publishedDigest}`)) fail(`${image.name} is missing from the publishing matrix`)
   }
   if ((policy?.blocked ?? []).length !== 1 || policy.blocked[0]?.name !== 'synapse' || !policy.blocked[0]?.reason) fail('Synapse must remain an explicit blocked candidate')
 
   const workflowRequirements = [
     'branches: [main, beta/production-readiness-2026-08]',
+    'default: false',
     'packages: write',
     'id-token: write',
     'attestations: write',
     'platforms: linux/amd64,linux/arm64',
     'push: true',
-    'provenance: mode=max',
+    'provenance: false',
     'sbom: false',
+    'staging-${{ github.run_id }}-${{ github.run_attempt }}',
+    'Candidate release tags are immutable',
+    'The staged build digest does not match the precomputed reviewed digest.',
+    'docker buildx imagetools create',
     'cosign attest --yes --type cyclonedx',
     'cosign sign --yes',
     'actions/attest-build-provenance@',
@@ -80,6 +89,20 @@ export function validateContainerCandidates({ policy, workflow, dockerfiles, pat
     'if: always()',
   ]
   for (const required of workflowRequirements) if (!workflow.includes(required)) fail(`candidate workflow is missing ${required}`)
+  const manualPublishGuard = "if: github.event_name == 'workflow_dispatch' && inputs.publish"
+  for (const stepName of [
+    'Refuse an existing release tag before publication',
+    'Sign in to GitHub Container Registry',
+    'Build and publish staged multi-platform candidate',
+    'Sign exact candidate digest with GitHub OIDC',
+    'Publish registry provenance attestation',
+    'Attach the pinned CycloneDX SBOM to the exact digest',
+  ]) {
+    const start = workflow.indexOf(`- name: ${stepName}`)
+    const next = workflow.indexOf('\n      - name:', start + 1)
+    const block = start >= 0 ? workflow.slice(start, next >= 0 ? next : undefined) : ''
+    if (!block.includes(manualPublishGuard)) fail(`${stepName} must remain manual-publication-only`)
+  }
   for (const action of ['actions/checkout', 'actions/setup-node', 'docker/setup-buildx-action', 'docker/login-action', 'docker/build-push-action', 'sigstore/cosign-installer', 'actions/attest-build-provenance', 'anchore/sbom-action', 'anchore/scan-action', 'actions/upload-artifact']) {
     if (!new RegExp(`${action.replace('/', '\\/')}@[0-9a-f]{40}`, 'u').test(workflow)) fail(`${action} must be SHA pinned`)
   }

mesh/scripts/check-container-candidates.test.mjs

@@ -46,3 +46,15 @@ test('rejects root runtimes and source patch drift', async () => {
   assert.match(errors, /USER 65532:65532/u)
   assert.match(errors, /source patch hash drifted/u)
 })
+
+test('rejects mutable tags, digest drift, and scheduled publication', async () => {
+  const input = await fixture()
+  input.policy.images[0].publishedDigest = 'sha256:invalid'
+  input.workflow = input.workflow
+    .replace('Candidate release tags are immutable', 'Candidate tags may move')
+    .replace("if: github.event_name == 'workflow_dispatch' && inputs.publish\n        uses: docker/login-action", 'uses: docker/login-action')
+  const errors = validateContainerCandidates(input).join('; ')
+  assert.match(errors, /published digest is invalid/u)
+  assert.match(errors, /Candidate release tags are immutable/u)
+  assert.match(errors, /Sign in to GitHub Container Registry must remain manual-publication-only/u)
+})