fix(ci): dispatch TestGrid from evidence ingest

Signed-off-by: Sujan Rao <sujan@nvidia.com>

Author: srao-nv

.github/workflows/evidence-ingest.yaml

@@ -233,42 +233,33 @@ jobs:
             done
 
   # ---------------------------------------------------------------------
-  # Job 3: refresh the published dashboard. Every successful ingest lands
-  # new evidence in GCS, so the GP5 dashboard (evidence-dashboard-publish)
-  # must re-render to surface it — otherwise the site is stale until the
-  # next unrelated push to main. The push-to-main ingest path already
-  # re-triggers the dashboard via its own `push:` trigger, but the
-  # first-party UAT path is a NESTED workflow_call (up to four levels deep:
-  # uat-run -> uat-{aws,gcp,azure} -> evidence-ingest), which emits no
-  # top-level run event and cannot nest a fifth reusable workflow. A
-  # workflow_dispatch decouples cleanly: it starts a fresh top-level run
-  # regardless of how deep this ingest was invoked.
+  # Job 3: trigger downstream evidence consumers. Every successful ingest lands
+  # verified evidence in GCS, so the static dashboard must re-render and the
+  # live TestGrid publisher must receive the same digest-pinned bundle ref.
   #
-  # Held in a SEPARATE job so the GCS-credentialed publish job never also
-  # carries actions:write. `needs: publish` inherits the produced gate — a
-  # skipped/failed publish skips this too, so the dashboard only re-renders
-  # after evidence is actually in the bucket. The dashboard's own
-  # concurrency group ("pages", cancel-in-progress:false) coalesces the
-  # flurry from a multi-cell nightly batch into a single queued rebuild
-  # that the newest dispatch supersedes.
+  # The first-party UAT path is a nested workflow_call (uat-run -> cloud UAT ->
+  # evidence-ingest), where workflow_run chaining is suppressed for nightly runs
+  # initiated by GITHUB_TOKEN. Explicit workflow_dispatch starts fresh top-level
+  # runs regardless of nesting depth or the original UAT actor.
   #
-  # The refresh is BEST-EFFORT: the evidence is already in the bucket and the
-  # dashboard self-heals on the next ingest or push to main, so a transient
-  # dispatch failure must never fail this workflow_call chain — otherwise the
-  # nightly controller would record a fully-successful, multi-hour UAT cell as
-  # a failed leg. The dispatch step retries, then warns (never fails), so
-  # staleness is surfaced but not fatal — the same not-cry-wolf posture as
-  # uat-superseded-notice.yaml.
+  # Held in a separate job so the GCS-credentialed publish job never also
+  # carries actions:write. `needs: publish` ensures downstream consumers are
+  # triggered only after verified evidence is persisted.
+  #
+  # Dispatch is best-effort: evidence is already durable, and a transient
+  # dispatch failure must not turn a successful multi-hour UAT cell red. Each
+  # dispatch retries and emits a visible warning on exhaustion. The dashboard
+  # self-heals on the next ingest; TestGrid can be backfilled by bundle ref.
   # ---------------------------------------------------------------------
-  trigger-dashboard:
-    name: Trigger dashboard publish
+  trigger-publishes:
+    name: Trigger evidence publishes
     needs: publish
     if: github.repository == 'nvidia/aicr'
     runs-on: ubuntu-latest
     timeout-minutes: 5
     permissions:
       contents: read
-      actions: write  # dispatch evidence-dashboard-publish.yaml
+      actions: write  # dispatch dashboard and TestGrid workflows
     steps:
       - name: Dispatch dashboard publish
         env:
@@ -293,3 +284,27 @@ jobs:
             sleep 5
           done
           echo "::warning::could not dispatch Evidence: Dashboard Publish after 3 attempts; the dashboard will refresh on the next ingest or push to main"
+
+      - name: Dispatch TestGrid publish
+        if: >-
+          inputs.bundle_ref != '' &&
+          (github.ref_name == 'main' || startsWith(github.ref_name, 'release/'))
+        env:
+          BUNDLE_REF: ${{ inputs.bundle_ref }}
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+        run: |
+          set -euo pipefail
+          for attempt in 1 2 3; do
+            if gh workflow run testgrid-publish.yml \
+              --repo "${REPO}" --ref main \
+              -f bundle_ref="${BUNDLE_REF}" \
+              -f source_class=uat \
+              -f environment=prod; then
+              echo "dispatched TestGrid Publish on main"
+              exit 0
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in 5s..."
+            sleep 5
+          done
+          echo "::warning::could not dispatch TestGrid Publish after 3 attempts; backfill bundle ${BUNDLE_REF} manually"

.github/workflows/testgrid-publish.yml

@@ -14,22 +14,20 @@
 
 # TestGrid Publish (TG5)
 #
-# Publishes UAT evidence bundles to the AICR TestGrid dashboard after
-# every completed UAT run (uat-run.yaml, which wraps the GCP or AWS
-# pipeline — any conclusion; a failure or cancelled run that never
-# reached conformance simply has nothing to publish and no-ops cleanly).
+# Publishes verified UAT evidence bundles to the AICR TestGrid dashboard.
+# Evidence ingest dispatches this workflow only after conformance produced and
+# verified a digest-pinned bundle.
 #
 # Disabled by default — gated on vars.TESTGRID_PUBLISH_ENABLED == 'true' —
 # until the prod WIF pool/SA exist (see aicr-testgrid Terraform apply).
 #
 # Pipeline:
-#   1. Validate the producer run is from this repository and branch
-#   2. Download the evidence pointer artifact from the triggering UAT run
-#   3. Extract + validate the OCI bundle ref + digest from pointer.yaml
-#   4. Authenticate to GCP via WIF (publish SA, groups/ prefix write-only)
-#   5. Log in to GHCR to pull the evidence OCI bundle
-#   6. Build testgrid-publish from source
-#   7. Pull the OCI bundle and write started/junit/finished to GCS
+#   1. Accept the verified, digest-pinned OCI bundle ref from evidence ingest
+#   2. Validate the bundle ref format
+#   3. Authenticate to GCP via WIF (publish SA, groups/ prefix write-only)
+#   4. Log in to GHCR to pull the evidence OCI bundle
+#   5. Build testgrid-publish from source
+#   6. Pull the OCI bundle and write started/junit/finished to GCS
 #
 # Auth:
 #   WIF pool:  aicr-testgrid[-<env>]-github   (aicr-testgrid Terraform)
@@ -56,48 +54,10 @@ permissions:
   contents: read
 
 on:
-  workflow_run:
-    # "UAT Run" (uat-run.yaml) is the only UAT workflow that always executes
-    # top-level: both a human (workflow_dispatch) and the nightly/daytime
-    # controllers trigger it via `gh workflow run` (a real dispatch, not
-    # `uses:`) specifically to keep it top-level — see the "Dispatch (not
-    # workflow_call)" comments in uat-nightly-batch.yaml / uat-daytime.yaml.
-    # uat-gcp.yaml / uat-aws.yaml are workflow_call-only reusable workflows
-    # invoked exclusively via `uses:` from uat-run.yaml's run-gcp/run-aws
-    # jobs — GitHub never emits a workflow_run event for a workflow that
-    # only ever executes as a workflow_call target, so watching them
-    # directly (as this file previously did) never fires.
-    #
-    # Jobs in a called reusable workflow share the caller's github.run_id,
-    # so github.event.workflow_run.id below still correctly matches the
-    # ${{ github.run_id }}-suffixed evidence-pointer artifact names
-    # uploaded from inside uat-gcp.yaml / uat-aws.yaml, and
-    # github.event.workflow_run.conclusion still reflects the real UAT
-    # outcome (run-gcp/run-aws call the reusable workflows with no
-    # continue-on-error, so a conformance failure propagates to
-    # uat-run.yaml's own conclusion).
-    workflows:
-      - "UAT Run"
-    types:
-      - completed
-  # Allow manual dispatch for testing and backfills.
-  #
-  # Trust model: the real enforcement is at the WIF layer, not the
-  # github.ref == 'refs/heads/main' check below (that check is just a
-  # fast, readable fail-closed guard, evaluated from whatever copy of this
-  # file was dispatched — it cannot be the actual security boundary).
-  # The publish SA's IAM binding (aicr-testgrid's wif-publish.tf,
-  # google_service_account_iam_member.publish_wif) grants
-  # roles/iam.workloadIdentityUser via a principalSet pinned to
-  # attribute.workflow_ref == ".../testgrid-publish.yml@refs/heads/main".
-  # GitHub's OIDC token embeds the actual ref the workflow file was loaded
-  # from as job_workflow_ref, so dispatching a modified copy of this file
-  # from any other branch produces a token that does not match the
-  # principalSet — WIF impersonation is rejected at the GCP layer
-  # regardless of what the dispatched copy's own if: checks say.
-  #
-  # bundle_ref is validated for format (ghcr.io/<org>/...@sha256:<64hex>)
-  # but not provenance/signature before the tool pulls and parses it.
+  # Evidence ingest explicitly dispatches this workflow with the verified,
+  # digest-pinned bundle ref. workflow_dispatch is an allowed GITHUB_TOKEN
+  # recursion exception, unlike workflow_run chains from bot-dispatched UAT.
+  # Manual dispatch uses the same input for testing and backfills.
   workflow_dispatch:
     inputs:
       bundle_ref:
@@ -120,14 +80,9 @@ on:
           - staging
 
 concurrency:
-  # One publish per triggering run (workflow_run) or per environment+bundle
-  # (workflow_dispatch) — prevents duplicate/racing columns if the workflow
-  # is re-run, or the same bundle_ref is dispatched twice, while a prior
-  # publish is still in flight. environment is included in the dispatch key
-  # so a staging backfill and a prod publish of the same bundle_ref (which
-  # write to different buckets) don't serialize behind each other. Falls
-  # back to run_id only when neither is available.
-  group: testgrid-publish-${{ github.event.workflow_run.id || format('{0}-{1}', inputs.environment, inputs.bundle_ref) || github.run_id }}
+  # Serialize publishes for the same environment and bundle while one is in
+  # flight. The publisher's create-only writes guard against later duplicates.
+  group: testgrid-publish-${{ format('{0}-{1}', inputs.environment, inputs.bundle_ref) }}
   cancel-in-progress: false  # never cancel an in-flight GCS write
 
 jobs:
@@ -136,29 +91,14 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15  # Go build ~2 min + gcloud uploads ~3 min; 15 min is generous
 
-    # Gate on vars.TESTGRID_PUBLISH_ENABLED so this workflow is a true no-op
-    # (skipped, not failed) until the prod WIF pool/SA exist in Terraform.
-    # Flip to 'true' once aicr-testgrid prod Terraform is applied — before
-    # that, every successful UAT would otherwise fail at the GCP auth step
-    # and produce CI noise.
-    #
-    # For workflow_run: run for ANY completed conclusion (success, failure,
-    # cancelled, timed_out, skipped, ...) — "Resolve bundle ref" already
-    # no-ops cleanly when no evidence pointer exists (only uploaded on a
-    # successful conformance phase), so non-terminal conclusions fall
-    # through to the same benign skip path instead of the job silently
-    # never running at all.
-    # For workflow_dispatch: restrict to main as a readable fail-closed
-    # guard — the actual security boundary is the WIF SA binding, not this
-    # check (see trust-model comment on the trigger above).
+    # Stay disabled until the prod WIF pool/SA exists. The workflow is always
+    # loaded from main; WIF's workflow_ref condition is the security boundary.
     if: >
       vars.TESTGRID_PUBLISH_ENABLED == 'true' &&
-      ((github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') ||
-       github.event_name == 'workflow_run')
+      github.ref == 'refs/heads/main'
 
     permissions:
       contents: read
-      actions: read    # required to download artifacts from the triggering run
       packages: read   # required to pull the evidence OCI bundle from ghcr.io
       id-token: write  # required for WIF token exchange
 
@@ -171,38 +111,6 @@ jobs:
         with:
           persist-credentials: false
 
-      # Sanity check before any credential exchange: confirm the producer
-      # run belongs to this repository. Note: github.event.workflow_run.
-      # repository is always the base repo that hosts the workflow (never
-      # a fork) — a repo-scoped workflow_run trigger cannot fire from a
-      # fork's own run in the first place, so this is defense-in-depth,
-      # not fork rejection. WIF's attribute_condition in wif-publish.tf is
-      # the actual security boundary; this check just fails fast with a
-      # clear message instead of a cryptic WIF rejection.
-      - name: Validate producer
-        if: github.event_name == 'workflow_run'
-        env:
-          PRODUCER_REPO: ${{ github.event.workflow_run.repository.full_name }}
-          PRODUCER_BRANCH: ${{ github.event.workflow_run.head_branch }}
-        run: |
-          set -euo pipefail
-          if [ "${PRODUCER_REPO}" != "${GITHUB_REPOSITORY}" ]; then
-            echo "::error::Producer workflow is from '${PRODUCER_REPO}', expected '${GITHUB_REPOSITORY}' — rejecting"
-            exit 1
-          fi
-          # Allowlist: main and release/* branches qualify for TestGrid.
-          # UAT supports workflow_dispatch on any branch, so without this
-          # an experimental dev-branch UAT would land a column in the
-          # dashboard. Reject anything else explicitly (fail closed).
-          case "${PRODUCER_BRANCH}" in
-            main|release/*)
-              ;;
-            *)
-              echo "::error::Producer ran on branch '${PRODUCER_BRANCH}' — only main and release/* branches publish to TestGrid"
-              exit 1
-              ;;
-          esac
-
       # Resolve environment-specific GCP resource names.
       # prod  → aicr-testgrid          (no env suffix — matches Terraform cluster_name)
       # other → aicr-testgrid-<env>
@@ -238,103 +146,21 @@ jobs:
         id: versions
         uses: ./.github/actions/load-versions
 
-      # Install yq (used to parse pointer.yaml) and Go using the repo's
-      # shared composite action — keeps tool versions in sync with .settings.yaml.
-      - name: Set up build tools
-        uses: ./.github/actions/setup-build-tools
-        with:
-          install_yq: 'true'
-          yq_version: ${{ steps.versions.outputs.yq }}
 
       - name: Set up Go
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e  # v7.0.0
         with:
           go-version: ${{ steps.versions.outputs.go }}
           cache: false
 
-      # Download the evidence pointer artifact from the triggering UAT run.
-      # The pointer is only uploaded when conformance succeeds — on failure runs
-      # with no pointer artifact the download action exits non-zero; we capture
-      # the outcome and distinguish "not found" (benign) from network errors.
-      - name: Download evidence pointer
-        id: download-pointer
-        if: github.event_name == 'workflow_run'
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
-        with:
-          github-token: ${{ github.token }}
-          run-id: ${{ github.event.workflow_run.id }}
-          pattern: "*-evidence-pointer-${{ github.event.workflow_run.id }}"
-          merge-multiple: true
-          path: evidence-pointer/
-        continue-on-error: true  # outcome checked explicitly in Resolve bundle ref
-
-      # Extract the OCI ref + digest from pointer.yaml.
-      # For workflow_dispatch the bundle_ref input is used directly.
-      # All user-controlled values flow through env — never inline ${{ }}
-      # in shell scripts to prevent script injection.
-      #
-      # pointer.yaml schema:
-      #   attestations:
-      #     - bundle:
-      #         oci: ghcr.io/nvidia/aicr-uat-gke-h100-training-<run_id>
-      #         digest: sha256:abc123...
+      # Validate the digest-pinned OCI ref supplied by evidence ingest or a
+      # manual backfill. Pass it through env rather than inline shell expansion.
       - name: Resolve bundle ref
         id: bundle
         env:
-          EVENT_NAME: ${{ github.event_name }}
-          DISPATCH_BUNDLE_REF: ${{ inputs.bundle_ref }}
-          DOWNLOAD_POINTER_OUTCOME: ${{ steps.download-pointer.outcome }}
-          UAT_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
+          BUNDLE_REF: ${{ inputs.bundle_ref }}
         run: |
           set -euo pipefail
-          if [ "${EVENT_NAME}" = "workflow_dispatch" ]; then
-            BUNDLE_REF="${DISPATCH_BUNDLE_REF}"
-          else
-            # If the download step failed, distinguish "artifact not found"
-            # (conformance did not complete — benign skip) from a real error
-            # (network failure — should not silently drop the publish).
-            # `-print -quit` + `|| true` tolerates an absent evidence-pointer/
-            # directory under set -euo pipefail (find would otherwise exit
-            # non-zero and kill the script before the -z check below runs).
-            POINTER="$(find evidence-pointer/ -name 'pointer.yaml' -print -quit 2>/dev/null || true)"
-            DOWNLOAD_OUTCOME="${DOWNLOAD_POINTER_OUTCOME}"
-            if [ -z "${POINTER}" ]; then
-              if [ "${DOWNLOAD_OUTCOME}" = "failure" ] && [ "${UAT_CONCLUSION}" = "success" ]; then
-                # Narrow known edge case: a manual `skip_tests` UAT dispatch
-                # also concludes success with no pointer (conformance never
-                # ran), which looks identical to a network error from here.
-                # Accepted — this only affects operator-triggered skip_tests
-                # runs, which have nothing to publish anyway.
-                echo "::error::UAT succeeded but evidence pointer download failed — possible network error"
-                exit 1
-              fi
-              if [ "${DOWNLOAD_OUTCOME}" = "failure" ]; then
-                # Unlike the success case above, a non-success UAT conclusion
-                # can't be disambiguated: artifact absence is expected here
-                # (conformance may never have run), so a genuine transient
-                # download error looks identical. Surface it for visibility
-                # without failing the job — there's nothing to publish
-                # either way.
-                echo "::warning::Evidence pointer download failed for a non-success UAT run (conclusion=${UAT_CONCLUSION}) — could be genuine absence or a transient error; treating as skip"
-              fi
-              echo "::notice::No evidence pointer found (conformance did not complete) — skipping TestGrid publish"
-              echo "skip=true" >> "${GITHUB_OUTPUT}"
-              exit 0
-            fi
-            OCI=$(yq '.attestations[0].bundle.oci' "${POINTER}")
-            DIGEST=$(yq '.attestations[0].bundle.digest' "${POINTER}")
-            if [ -z "${OCI}" ] || [ "${OCI}" = "null" ]; then
-              echo "::error::Could not extract OCI ref from ${POINTER}"
-              cat "${POINTER}"
-              exit 1
-            fi
-            if [ -z "${DIGEST}" ] || [ "${DIGEST}" = "null" ]; then
-              echo "::error::Could not extract digest from ${POINTER}"
-              cat "${POINTER}"
-              exit 1
-            fi
-            BUNDLE_REF="${OCI}@${DIGEST}"
-          fi
           # Validate format: ghcr.io/<org>/...@sha256:<64 hex chars>
           # Namespace is intentionally NOT restricted to nvidia/ — community
           # bundles (source_class=community) are hosted under contributor