UAT kind-h100 training @ v0.19.0 #32689265711-1-kind-h100-3 #936

Workflow file for this run

.github/workflows/uat-run.yaml at 709b179

	# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	name: UAT Run

	# A distinct run title per dispatch so the nightly controller can identify the
	# run it dispatched — `gh workflow run` does not return a run id, so the
	# controller matches on this display title. The controller supplies a unique
	# dispatch_key (its run id + cell index) that is appended here, so a concurrent
	# manual dispatch of the same reservation/version can never be mistaken for the
	# controller's run. A manual dispatch omits the key and the suffix is dropped.
	# Quoted so YAML does not treat the ` #` in the dispatch_key suffix as a comment.
	#
	# The intent (training\|inference) is rendered so nightly/daytime runs are
	# self-describing in the Actions list. It defaults to `training` on both the
	# dispatch and call paths, so it is always present. The deployer is rendered
	# as `[<deployer>]` ONLY when it is non-empty and non-default (i.e. argocd
	# today) so nightly/daytime titles, which never dispatch a non-default
	# deployer, are unchanged. IMPORTANT: the nightly (uat-nightly-batch.yaml)
	# and daytime (uat-daytime.yaml) controllers rebuild this exact title to
	# resolve the run they dispatched — keep their `title=` construction in
	# lockstep with this format string; if either controller ever starts passing
	# `-f deployer=<x>` (x != helmfile), also add the `[<x>]` segment there.
	# Folded scalar (>-): each newline below joins as a single space, which
	# corresponds exactly to the literal single-space separators between the
	# segments (`UAT · reservation · intent · @ · version`). Line-break points
	# are chosen so no newline lands INSIDE an adjacent expression pair where
	# the absence of a literal separator is load-bearing (e.g. the intent+
	# deployer pair — deployer renders ' [<x>]' or '', its leading space is the
	# separator on its own — and version+dispatch_key, same shape). Keeps the
	# rendered title identical to the pre-fold single-line form.
	run-name: >-
	UAT ${{ inputs.reservation }}
	${{ inputs.intent }}${{ inputs.deployer != 'helmfile' && format(' [{0}]', inputs.deployer) \|\| '' }}
	@ ${{ inputs.aicr_version \|\| 'main' }}${{ inputs.dispatch_key != '' && format(' #{0}', inputs.dispatch_key) \|\| '' }}

	# The shared UAT dispatch surface and reservation-lease owner (#1274, DC1).
	# A human (workflow_dispatch) or the nightly batch (workflow_call) requests a
	# run against a named reservation; this workflow holds the per-reservation
	# concurrency lease, resolves the reservation row from the checked-in registry
	# (infra/uat/reservations.yaml) via the uat-broker helper, and dispatches the
	# cloud-appropriate reusable pipeline. Because every run for a reservation
	# routes through here and shares the lease, contending runs QUEUE instead of
	# racing the hardware.
	#
	# DC2 (#1275) adds two orthogonal dials, both forwarded to the cloud pipeline:
	# - intent (training\|inference) selects the per-intent test config +
	# recipe criteria.
	# - lifecycle (nightly\|daytime-up\|daytime-down) selects the cluster
	# lifecycle: the nightly provision→CUJ→teardown, the daytime
	# provision-and-hold (stable lease-tagged name; holds on success, tears
	# down on failure or cancellation), or the evening teardown of that held
	# daytime cluster.
	on:
	workflow_dispatch:
	inputs:
	reservation:
	description: 'Reservation name from infra/uat/reservations.yaml (e.g. aws-h100, gcp-h100, azure-h100).'
	type: string
	required: true
	aicr_version:
	description: 'Released aicr version to test (e.g. v1.2.3); empty = build from source (main tip).'
	type: string
	default: ''
	dispatch_key:
	description: 'Unique correlation key set by the nightly controller so it can find this run; leave empty for manual runs.'
	type: string
	default: ''
	intent:
	description: 'Recipe intent — selects the per-intent test config (h100-<intent>-config.yaml) and recipe criteria.'
	type: choice
	options: [training, inference]
	default: training
	lifecycle:
	description: 'Cluster lifecycle. nightly: provision→CUJ→teardown. daytime-up: stable-named cluster, holds on success, tears down on failure or cancellation. daytime-down: tear down cluster.'
	type: choice
	options: [nightly, daytime-up, daytime-down]
	default: nightly
	slot:
	description: 'Daytime cluster slot (ADR-017) — the <slot> in aicr-uat-day-<slug>-<slot>-. Only slot 0 is provisioned today.'
	type: string
	default: '0'
	skip_delete:
	description: 'Skip cluster teardown (manual debugging only; ignored for daytime lifecycles). A retained cluster becomes eligible for janitor reclamation ~24h after the run finishes.'
	type: boolean
	default: false
	skip_tests:
	description: 'Skip UAT test execution (manual debugging only).'
	type: boolean
	default: false
	deployer:
	# helmfile = default via <accelerator>-<intent>-config.yaml (the
	# existing filename shape every cell has always used, kept for
	# backwards-compat). argocd = <accelerator>-<intent>-argocd-config.yaml
	# (e.g. h100-training-argocd-config.yaml). Only aws-h100 training
	# ships an argocd variant today; other cells fail with a missing
	# test-config. See issue #2194. (actionlint rejects an empty
	# string in a choice's options, so `helmfile` is the explicit
	# default value here; the TEST_CONFIG resolver in uat-aws.yaml
	# maps it to the unsuffixed filename.)
	description: 'Deployer variant. See workflow file for the variant matrix.'
	type: choice
	options: [helmfile, argocd]
	default: helmfile
	workflow_call:
	inputs:
	reservation:
	type: string
	required: true
	aicr_version:
	type: string
	default: ''
	dispatch_key:
	type: string
	default: ''
	intent:
	type: string
	default: training
	lifecycle:
	type: string
	default: nightly
	slot:
	type: string
	default: '0'
	skip_delete:
	type: boolean
	default: false
	skip_tests:
	type: boolean
	default: false
	deployer:
	type: string
	default: helmfile

	permissions:
	contents: read

	# The reservation lease. Every run targeting a given reservation serializes on
	# this group, so a contender waits rather than racing the reservation;
	# cancel-in-progress:false keeps an in-progress run (and its teardown) from
	# being killed.
	#
	# NOTE: this is a single-slot lease, not a full queue. GitHub concurrency holds
	# at most one in-progress + one PENDING run per group; a third contender
	# supersedes (cancels) the older pending one — cancel-in-progress:false only
	# protects the in-progress run, not the pending slot. Acceptable at launch (two
	# reservations, at most cron + one ad-hoc each); preserving every request would
	# need the deferred standing broker (see docs/contributor/uat.md, #1264).
	#
	# github.event.inputs is used for the dispatch path because the inputs context
	# is unreliable in a top-level concurrency group on workflow_dispatch
	# (community #45734/#35341); inputs.* covers the workflow_call path, where
	# github.event.inputs is empty.
	concurrency:
	group: uat-${{ github.event.inputs.reservation \|\| inputs.reservation }}
	cancel-in-progress: false

	jobs:
	# Resolve the reservation row to the cloud + on-disk config paths the
	# cloud-specific pipeline consumes. A typo'd reservation name fails here
	# (uat-broker exits NOT_FOUND), so the registry is the single source of truth.
	resolve:
	if: github.repository == 'nvidia/aicr'
	runs-on: ubuntu-latest
	outputs:
	cloud: ${{ steps.row.outputs.cloud }}
	# Daytime cluster-name discovery slug (ADR-017); forwarded to the cloud
	# pipelines that derive aicr-uat-day-<slug>-<slot>-<run_id>.
	slug: ${{ steps.row.outputs.slug }}
	cluster_config_path: ${{ steps.row.outputs['cluster-config-path'] }}
	test_config_dir: ${{ steps.row.outputs['test-config-dir'] }}
	# Accelerator selects the per-accelerator test-config filename
	# (<accelerator>-<intent>-config.yaml) in the AWS pipeline. The AWS
	# account (tenancy) is NOT plumbed here: the AWS pipeline derives it from
	# the cluster-config's .deployment.tenancy, the single source of truth.
	accelerator: ${{ steps.row.outputs.accelerator }}
	steps:
	- name: Checkout
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- name: Load versions
	id: versions
	uses: ./.github/actions/load-versions
	- name: Setup Go
	uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
	with:
	go-version: '${{ steps.versions.outputs.go }}'
	cache: true
	cache-dependency-path: \|
	go.sum
	vendor/modules.txt
	- name: Resolve reservation row
	id: row
	env:
	GOFLAGS: -mod=vendor
	# Pass via env (not inline interpolation) to avoid expression
	# injection through the dispatch input.
	RESERVATION: ${{ inputs.reservation }}
	SLOT: ${{ inputs.slot }}
	run: \|
	set -euo pipefail
	# Only slot 0 is provisioned today (ADR-017). Reject any other value at
	# this single dispatch surface — before it is forwarded to a cloud
	# pipeline — so a daytime-up with slot=1 cannot stand up a second daytime
	# cluster (and an unbounded value cannot blow the GKE name budget) until
	# multi-slot capacity and scheduling land.
	if [[ "${SLOT}" != "0" ]]; then
	echo "::error::unsupported slot '${SLOT}' (only slot 0 is enabled today)"; exit 1
	fi
	go build -o ./bin/uat-broker ./tools/uat-broker
	./bin/uat-broker reservations --name "$RESERVATION" >> "$GITHUB_OUTPUT"

	# One of run-aws / run-gcp / run-azure / run-kind fires based on the resolved
	# cloud (the registry validates cloud membership, so at most one matches; the
	# unmapped-cloud job below fails closed if a registry cloud has no
	# run-<cloud> job here yet). The caller job grants the permission SUPERSET
	# its callee needs — a reusable workflow's jobs are ceilinged by the calling
	# job's permissions, and the called pipeline does OIDC, GHCR push, and
	# (nested) evidence-ingest GCS WIF.
	run-aws:
	needs: resolve
	if: always() && needs.resolve.outputs.cloud == 'aws'
	# Superset ceiling for the nested pipeline (a reusable workflow's jobs
	# cannot exceed the caller's permissions).
	permissions:
	contents: read # checkout inside the called pipeline
	actions: write # nested uat-{aws,gcp} watchers + evidence-ingest dashboard dispatch
	id-token: write # cloud OIDC/WIF: provision, keyless cosign, and GCS evidence publish
	packages: write # push validator + snapshot-agent images to GHCR
	uses: ./.github/workflows/uat-aws.yaml
	with:
	reservation: ${{ inputs.reservation }}
	aicr_version: ${{ inputs.aicr_version }}
	intent: ${{ inputs.intent }}
	lifecycle: ${{ inputs.lifecycle }}
	# Daytime cluster-name axes (ADR-017): slug from the resolved registry row,
	# slot from the dispatch input. Together with github.run_id they form
	# aicr-uat-day-<slug>-<slot>-<run_id>.
	slug: ${{ needs.resolve.outputs.slug }}
	slot: ${{ inputs.slot }}
	cluster_config_path: ${{ needs.resolve.outputs.cluster_config_path }}
	test_config_dir: ${{ needs.resolve.outputs.test_config_dir }}
	accelerator: ${{ needs.resolve.outputs.accelerator }}
	skip_delete: ${{ inputs.skip_delete }}
	skip_tests: ${{ inputs.skip_tests }}
	deployer: ${{ inputs.deployer }}

	run-gcp:
	needs: resolve
	# deployer=argocd is aws-h100-scoped today (issue #2194): uat-gcp.yaml declares
	# no deployer input, so passing one here would be silently ignored (green
	# helmfile run against a request the user did not make). Skip this job when
	# a non-helmfile deployer is dispatched; the unsupported-deployer-for-cloud
	# guard job below emits the red-workflow signal for that combination.
	if: always() && needs.resolve.outputs.cloud == 'gcp' && inputs.deployer == 'helmfile'
	# Superset ceiling for the nested pipeline (a reusable workflow's jobs
	# cannot exceed the caller's permissions).
	permissions:
	contents: read # checkout inside the called pipeline
	actions: write # nested uat-{aws,gcp} watchers + evidence-ingest dashboard dispatch
	id-token: write # cloud OIDC/WIF: provision, keyless cosign, and GCS evidence publish
	packages: write # push validator + snapshot-agent images to GHCR
	uses: ./.github/workflows/uat-gcp.yaml
	with:
	reservation: ${{ inputs.reservation }}
	aicr_version: ${{ inputs.aicr_version }}
	intent: ${{ inputs.intent }}
	lifecycle: ${{ inputs.lifecycle }}
	# Daytime cluster-name axes (ADR-017): slug from the resolved registry row,
	# slot from the dispatch input. Together with github.run_id they form
	# aicr-uat-day-<slug>-<slot>-<run_id>.
	slug: ${{ needs.resolve.outputs.slug }}
	slot: ${{ inputs.slot }}
	cluster_config_path: ${{ needs.resolve.outputs.cluster_config_path }}
	test_config_dir: ${{ needs.resolve.outputs.test_config_dir }}
	skip_delete: ${{ inputs.skip_delete }}
	skip_tests: ${{ inputs.skip_tests }}

	run-azure:
	needs: resolve
	# See run-gcp for the deployer-scope rationale.
	if: always() && needs.resolve.outputs.cloud == 'azure' && inputs.deployer == 'helmfile'
	# Superset ceiling for the nested pipeline (a reusable workflow's jobs
	# cannot exceed the caller's permissions).
	permissions:
	contents: read # checkout inside the called pipeline
	actions: write # nested uat-* watchers + evidence-ingest dashboard dispatch
	id-token: write # cloud OIDC/WIF: provision, keyless cosign, and GCS evidence publish
	packages: write # push validator + snapshot-agent images to GHCR
	uses: ./.github/workflows/uat-azure.yaml
	with:
	reservation: ${{ inputs.reservation }}
	aicr_version: ${{ inputs.aicr_version }}
	intent: ${{ inputs.intent }}
	lifecycle: ${{ inputs.lifecycle }}
	# Daytime cluster-name axes (ADR-017): slug from the resolved registry row,
	# slot from the dispatch input. Together with github.run_id they form
	# aicr-uat-day-<slug>-<slot>-<run_id>.
	slug: ${{ needs.resolve.outputs.slug }}
	slot: ${{ inputs.slot }}
	cluster_config_path: ${{ needs.resolve.outputs.cluster_config_path }}
	test_config_dir: ${{ needs.resolve.outputs.test_config_dir }}
	skip_delete: ${{ inputs.skip_delete }}
	skip_tests: ${{ inputs.skip_tests }}

	# nvkind real-silicon lane (DC5 #1278). "kind" is not a cloud but a
	# self-hosted GPU-runner lane; it slots into the same dispatch model so it
	# rides the nightly batch and shares tests/uat/kind/run + tests/uat/lib. It
	# provisions on the runner (no cloud actuator), so it needs no cloud OIDC —
	# but it DOES do keyless cosign (id-token), GHCR push (packages), and the
	# nested evidence-ingest GCS WIF, so the permission superset matches the
	# cloud lanes.
	run-kind:
	needs: resolve
	# See run-gcp for the deployer-scope rationale.
	if: needs.resolve.outputs.cloud == 'kind' && inputs.deployer == 'helmfile'
	permissions:
	contents: read # checkout inside the called pipeline
	actions: write # nested evidence-ingest dashboard-publish dispatch (a reusable workflow cannot elevate the caller token, so grant the ceiling here)
	id-token: write # keyless cosign + GCS evidence-publish WIF
	packages: write # push validator/agent + evidence images to GHCR
	uses: ./.github/workflows/uat-kind.yaml
	with:
	reservation: ${{ inputs.reservation }}
	aicr_version: ${{ inputs.aicr_version }}
	intent: ${{ inputs.intent }}
	lifecycle: ${{ inputs.lifecycle }}
	cluster_config_path: ${{ needs.resolve.outputs.cluster_config_path }}
	test_config_dir: ${{ needs.resolve.outputs.test_config_dir }}
	skip_delete: ${{ inputs.skip_delete }}
	skip_tests: ${{ inputs.skip_tests }}

	# Fail closed on a dispatched deployer variant this cell doesn't yet wire.
	# Today only run-aws forwards the `deployer` input to its reusable pipeline;
	# gcp/azure/kind reusable workflows declare no such input and would run the
	# helmfile lane if we let them fire. The run-*.if:s above skip those jobs
	# when a non-helmfile deployer is dispatched to a non-aws cell; this job
	# provides the paired red-workflow signal so the requester never sees a
	# green run for a request that was silently downgraded.
	# Mirrors the unmapped-cloud pattern for the same "no silent no-op" reason.
	unsupported-deployer-for-cloud:
	needs: resolve
	if: >-
	inputs.deployer != 'helmfile' &&
	needs.resolve.outputs.cloud != 'aws'
	runs-on: ubuntu-latest
	steps:
	- name: Fail on unsupported (deployer, cloud) combination
	env:
	CLOUD: ${{ needs.resolve.outputs.cloud }}
	DEPLOYER: ${{ inputs.deployer }}
	run: \|
	set -euo pipefail
	echo "::error::deployer='${DEPLOYER}' is only wired for cloud='aws' today"
	echo "::error::(this reservation resolved to cloud='${CLOUD}'). Only run-aws"
	echo "::error::forwards the deployer input to its reusable pipeline; other"
	echo "::error::run-<cloud> jobs would run the helmfile lane silently. See"
	echo "::error::issue #2194 for the coverage roadmap. The run-<cloud>.if:"
	echo "::error::guards above skipped the dispatched cell to prevent that"
	echo "::error::silent downgrade."
	exit 1

	# Fail closed on an unmapped cloud. The broker validates cloud membership
	# against its own set, but a cloud can be added there (and to the registry)
	# before its run-<cloud> job exists here — without this job the dispatch
	# would silently no-op: the lease is taken, no pipeline runs, and the
	# requester sees a green run that did nothing.
	unmapped-cloud:
	needs: resolve
	if: >-
	needs.resolve.outputs.cloud != 'aws' &&
	needs.resolve.outputs.cloud != 'gcp' &&
	needs.resolve.outputs.cloud != 'azure' &&
	needs.resolve.outputs.cloud != 'kind'
	runs-on: ubuntu-latest
	steps:
	- name: Fail on unmapped cloud
	env:
	CLOUD: ${{ needs.resolve.outputs.cloud }}
	run: \|
	set -euo pipefail
	echo "::error::reservation resolved to cloud '${CLOUD}', which has no run-<cloud> job in uat-run.yaml — add one (see run-azure for the shape)."
	exit 1

Navigation Menu

Uh oh!

UAT kind-h100 training @ v0.19.0 #32689265711-1-kind-h100-3 #936

Workflow file

UAT kind-h100 training @ v0.19.0 #32689265711-1-kind-h100-3 #936

Uh oh!

Workflow file for this run