UAT Nightly Batch #42

Workflow file for this run

.github/workflows/uat-nightly-batch.yaml at 8703d8c

	# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	name: UAT Nightly Batch

	# The night side of the day/night cycle (#1274, DC1 + DC5): on a nightly cron,
	# run the version matrix per reservation. For each reservation the controller
	# expands an ordered, time-boxed schedule — main first, then the previous N
	# stable releases in descending semver order (uat-broker schedule) — and runs
	# the cells SEQUENTIALLY, dispatching the shared uat-run.yaml per cell and
	# waiting for it before the next. Different reservations run in parallel
	# (independent hardware); uat-run.yaml's per-reservation lease serializes runs
	# that share a reservation. When the time-box closes, remaining cells are
	# dropped oldest-release-first (they are ordered oldest-last, so the controller
	# simply stops). Dispatch (not workflow_call) keeps uat-run.yaml top-level, so
	# its own lease + superseded observer apply and the reusable-nesting depth stays
	# at 3 (uat-run -> uat-{aws,gcp} -> evidence-ingest).
	#
	# aicr_version is threaded per cell; the release cells install the released
	# aicr + validator/agent images at that version (DC5), main builds from source.
	on:
	schedule:
	- cron: '0 4 * * *' # 04:00 UTC daily (evening in US Pacific); default branch only
	workflow_dispatch:
	inputs:
	previous_n:
	description: 'Previous stable releases below main to run per reservation (0 = main only).'
	type: string
	default: '1'
	deadline_offset_hours:
	description: 'Time-box: hours after batch start to stop dispatching cells. Secondary cap — the drive also refuses any cell that cannot finish before its own timeout (see max_cell_minutes).'
	type: string
	default: '5'
	max_cell_minutes:
	description: 'Wall-clock a dispatched cell may need. The drive stops dispatching once fewer minutes than this remain before its timeout, so the last cell finishes cleanly.'
	type: string
	default: '150'

	permissions:
	contents: read

	# Serialize batch runs: a manual dispatch should not fan out a second batch
	# alongside the cron one. (The per-reservation leases in uat-run.yaml would
	# queue the duplicates anyway, but there is no reason to start them.)
	concurrency:
	group: uat-nightly-batch
	cancel-in-progress: false

	jobs:
	# Enumerate reservation names from the registry into a JSON array for the
	# matrix below — keeps the batch data-driven (no per-reservation YAML).
	enumerate:
	if: github.repository == 'nvidia/aicr'
	runs-on: ubuntu-latest
	outputs:
	reservations: ${{ steps.list.outputs.reservations }}
	# Single batch-start timestamp shared by every matrix leg, so the time-box
	# deadline is identical regardless of when each leg's runner starts.
	batch_start: ${{ steps.list.outputs.batch_start }}
	steps:
	- name: Checkout
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- name: Load versions
	id: versions
	uses: ./.github/actions/load-versions
	- name: Setup Go
	uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
	with:
	go-version: '${{ steps.versions.outputs.go }}'
	cache: true
	cache-dependency-path: \|
	go.sum
	vendor/modules.txt
	- name: Enumerate reservations
	id: list
	env:
	GOFLAGS: -mod=vendor
	run: \|
	set -euo pipefail
	go build -o ./bin/uat-broker ./tools/uat-broker
	mapfile -t names < <(./bin/uat-broker reservations --list)
	if [ "${#names[@]}" -eq 0 ]; then
	echo "::error::no reservations found in infra/uat/reservations.yaml"
	exit 1
	fi
	json=$(printf '%s\n' "${names[@]}" \| jq -R . \| jq -cs .)
	echo "reservations=${json}" >> "$GITHUB_OUTPUT"
	echo "Reservations: ${json}"
	# Capture the batch start once; every leg derives its deadline from this.
	echo "batch_start=$(date +%s)" >> "$GITHUB_OUTPUT"

	# One matrix leg per reservation (parallel). Each leg drives its own version
	# matrix sequentially. It only DISPATCHES uat-run.yaml (workflow_dispatch is
	# exempt from the GITHUB_TOKEN recursion rule and always creates a run), so
	# this job needs actions:write to dispatch + read to watch; the dispatched
	# uat-run.yaml runs top-level with its own permissions.
	drive:
	needs: enumerate
	runs-on: ubuntu-latest
	# The controller blocks on `gh run watch` per cell sequentially, so this job
	# runs as long as the batch. GitHub hard-caps a hosted job at 6h; set an
	# explicit timeout just under that so the job ends cleanly. The dispatch loop
	# is budget-aware: it stops dispatching once fewer than `max_cell_minutes`
	# remain before this timeout, so the last cell always finishes and the leg
	# drops-oldest gracefully instead of being killed mid-cell (issue #1778).
	# DRIVE_JOB_TIMEOUT_MINUTES in the run step below MUST equal this literal.
	timeout-minutes: 350
	strategy:
	fail-fast: false
	matrix:
	reservation: ${{ fromJSON(needs.enumerate.outputs.reservations) }}
	permissions:
	contents: read # checkout + build uat-broker
	actions: write # dispatch uat-run.yaml and watch the dispatched runs
	steps:
	- name: Checkout
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	fetch-depth: 0 # full tag history for `git tag` -> uat-broker schedule
	- name: Load versions
	id: versions
	uses: ./.github/actions/load-versions
	- name: Setup Go
	uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
	with:
	go-version: '${{ steps.versions.outputs.go }}'
	cache: true
	cache-dependency-path: \|
	go.sum
	vendor/modules.txt
	- name: Run version-matrix cells (sequential, time-boxed)
	env:
	GH_TOKEN: ${{ github.token }}
	GOFLAGS: -mod=vendor
	REPO: ${{ github.repository }}
	REF: ${{ github.ref_name }}
	RESERVATION: ${{ matrix.reservation }}
	PREVIOUS_N: ${{ inputs.previous_n \|\| '1' }}
	DEADLINE_OFFSET_HOURS: ${{ inputs.deadline_offset_hours \|\| '5' }}
	MAX_CELL_MINUTES: ${{ inputs.max_cell_minutes \|\| '150' }}
	# MUST equal the drive job's timeout-minutes literal above. Used to
	# derive the budget-aware dispatch cutoff so the last cell finishes
	# before GitHub kills this job (issue #1778).
	DRIVE_JOB_TIMEOUT_MINUTES: '350'
	BATCH_START: ${{ needs.enumerate.outputs.batch_start }}
	run: \|
	set -euo pipefail
	# Approximate this drive job's start (its timeout-minutes clock anchor).
	# Captured at the top of the first substantive step; the checkout/setup
	# steps before it already consumed a few minutes of the timeout budget,
	# so the real GitHub kill is slightly EARLIER than DRIVE_START + timeout.
	# SETUP_SLACK_MINUTES absorbs that skew (plus gh dispatch/watch latency)
	# so the budget-aware cutoff holds even for a full-length cell. It must
	# exceed this job's pre-`run` setup — a heavier-than-usual `fetch-depth:
	# 0` checkout + setup-go — or the mid-cell kill this guards against
	# returns; 10m gives ample margin over that few-minute setup. If setup
	# ever grows past it, switch DRIVE_START to the job's exact start_at:
	# gh api "repos/${REPO}/actions/runs/${GITHUB_RUN_ID}/jobs" \
	# --jq '.jobs[] \| select(.name\|endswith("('"${RESERVATION}"')")) \| .started_at'
	# (leaving the slack to cover only dispatch/watch latency).
	DRIVE_START=$(date +%s)
	SETUP_SLACK_MINUTES=10
	# Poll budget for finding a dispatched run (gh workflow run returns no
	# run id): POLL_MAX_ATTEMPTS x POLL_INTERVAL_SECONDS seconds.
	POLL_MAX_ATTEMPTS=24
	POLL_INTERVAL_SECONDS=5
	go build -o ./bin/uat-broker ./tools/uat-broker

	# Nightly intents for this reservation (#1276, DC3). The batch runs
	# EACH listed intent (training and/or inference) as its own full CUJ
	# cell per version, dispatched SEQUENTIALLY through the shared
	# per-reservation lease — so both intents run nightly with no
	# contention and no second cron. The broker resolves an ABSENT
	# nightly-intents to the training default; an EXPLICIT empty list is
	# a bring-up opt-out (manual dispatch only) and resolves to an empty
	# value. Distinguish the two failure shapes: a MISSING key is a
	# broker/registry regression (fail closed); a present-but-EMPTY
	# value is the opt-out (skip this leg gracefully).
	ROW=$(./bin/uat-broker reservations --name "$RESERVATION")
	if ! grep -q '^nightly-intents=' <<<"$ROW"; then
	echo "::error::could not resolve nightly-intents for ${RESERVATION} (key missing from broker output)" >&2
	exit 1
	fi
	NIGHTLY_INTENTS_CSV=$(sed -n 's/^nightly-intents=//p' <<<"$ROW")
	if [[ -z "$NIGHTLY_INTENTS_CSV" ]]; then
	echo "::notice::Reservation ${RESERVATION} opts out of the nightly batch (nightly-intents: []); skipping this leg."
	exit 0
	fi
	# The actual per-cell intents come from the schedule below, which
	# applies the reservation's nightly-intent-min-versions gate per
	# version; this line is just the leg-level summary of what's enrolled.
	echo "Nightly intents for ${RESERVATION}: ${NIGHTLY_INTENTS_CSV} (release cells further gated by nightly-intent-min-versions)"

	# Validate previous_n before the broker consumes it. `--previous-n` is
	# an int flag so it rejects non-integers, but it would accept a
	# negative and mis-shape the schedule; reject non-integer or negative
	# values here too, symmetric with the deadline_offset_hours guard.
	if [[ ! "$PREVIOUS_N" =~ ^[0-9]+$ ]]; then
	echo "::error::previous_n must be a non-negative integer; got '${PREVIOUS_N}'." >&2
	exit 1
	fi

	# Ordered cells for this reservation: main first, then the previous-N
	# stable releases in descending semver order (oldest last).
	schedule=$(git tag -l 'v*' \| ./bin/uat-broker schedule \
	--reservations "$RESERVATION" --previous-n "$PREVIOUS_N")
	mapfile -t versions < <(jq -r --arg r "$RESERVATION" '.[$r][].aicr_version' <<<"$schedule")
	echo "Cells for ${RESERVATION}: ${versions[*]:-<none>}"

	# Reject a non-integer / negative deadline_offset_hours before the
	# arithmetic below — workflow_dispatch passes it as a free-form string
	# and Bash would silently accept e.g. -1, moving the cutoff before
	# batch start and dropping every cell immediately.
	if [[ ! "$DEADLINE_OFFSET_HOURS" =~ ^[0-9]+$ ]]; then
	echo "::error::deadline_offset_hours must be a non-negative integer; got '${DEADLINE_OFFSET_HOURS}'." >&2
	exit 1
	fi
	# Same guard for max_cell_minutes (free-form workflow_dispatch string):
	# a non-integer or 0 would mis-shape the budget-aware cutoff below.
	if [[ ! "$MAX_CELL_MINUTES" =~ ^[0-9]+$ \|\| "$MAX_CELL_MINUTES" -eq 0 ]]; then
	echo "::error::max_cell_minutes must be a positive integer; got '${MAX_CELL_MINUTES}'." >&2
	exit 1
	fi
	# A cell reserve that leaves no room before the drive-job timeout would
	# drop every cell immediately (an empty, silently-green batch). Reject
	# it: the operator must lower max_cell_minutes or the job needs a longer
	# timeout-minutes (bounded by GitHub's 6h hosted-job cap).
	if (( MAX_CELL_MINUTES + SETUP_SLACK_MINUTES >= DRIVE_JOB_TIMEOUT_MINUTES )); then
	echo "::error::max_cell_minutes+slack (${MAX_CELL_MINUTES}+${SETUP_SLACK_MINUTES}) must be < drive-job timeout (${DRIVE_JOB_TIMEOUT_MINUTES}m); no cell could dispatch." >&2
	exit 1
	fi

	# Effective dispatch cutoff = the EARLIER of two deadlines:
	# (1) the user time-box, anchored to the single batch-start timestamp
	# (captured once in enumerate) so every reservation leg shares one
	# cutoff regardless of when its runner started; and
	# (2) the budget-aware cutoff: stop dispatching once fewer than
	# MAX_CELL_MINUTES remain before THIS drive job's own
	# timeout-minutes kill (anchored to DRIVE_START). This guarantees
	# the last cell dispatched has a full cell budget to finish before
	# GitHub cancels the job, so an overrun sheds the oldest remaining
	# cell gracefully instead of hard-failing the leg mid-cell (#1778).
	# (2) is per-leg (DRIVE_START differs across matrix legs); (1) is shared.
	offset_deadline=$(( BATCH_START + DEADLINE_OFFSET_HOURS * 3600 ))
	budget_deadline=$(( DRIVE_START + (DRIVE_JOB_TIMEOUT_MINUTES - MAX_CELL_MINUTES - SETUP_SLACK_MINUTES) * 60 ))
	deadline=$(( offset_deadline < budget_deadline ? offset_deadline : budget_deadline ))
	if (( budget_deadline < offset_deadline )); then
	echo "Dispatch cutoff: budget-aware (${MAX_CELL_MINUTES}m cell reserve before the ${DRIVE_JOB_TIMEOUT_MINUTES}m drive-job timeout), $(( (deadline - DRIVE_START) / 60 ))m into this leg."
	else
	echo "Dispatch cutoff: user time-box (deadline_offset_hours=${DEADLINE_OFFSET_HOURS}), $(( (deadline - BATCH_START) / 60 ))m after batch start."
	fi
	cell=0
	leg_failed=""
	# Version outer-loop, intent inner-loop: `main` runs every intent
	# before any release cell, so a time-box drop always sheds the OLDEST
	# release cells first — never main's inference. Each (version × intent)
	# is a full provision→CUJ→teardown dispatched through the shared lease,
	# so the intents serialize automatically (no contention).
	#
	# Intents are per-CELL: the schedule already applied each
	# reservation's nightly-intent-min-versions gate, so a release that
	# predates an intent's minimum version carries a shorter (or empty)
	# .intents list and that (version × intent) never dispatches; `main`
	# always carries every listed intent.
	for ver in "${versions[@]}"; do
	mapfile -t cell_intents < <(jq -r --arg r "$RESERVATION" --arg v "$ver" \
	'.[$r][] \| select(.aicr_version == $v) \| .intents[]' <<<"$schedule")
	# A fully-gated release cell (no eligible intents) contributes
	# nothing — skip it without consuming the time-box or cell counter.
	(( ${#cell_intents[@]} == 0 )) && continue
	for intent in "${cell_intents[@]}"; do
	if (( $(date +%s) >= deadline )); then
	echo "::notice::Dispatch cutoff reached (time-box or drive-job budget) — dropping remaining cells for ${RESERVATION} (oldest release first)."
	break 2
	fi
	cell=$((cell + 1))
	label="${ver:-main}"
	# Unique per-dispatch key (controller run id + attempt + reservation
	# + cell index) so the resolver below watches THIS run and never a
	# concurrent manual dispatch of the same reservation/version. The
	# cell index increments per (version × intent), so the two intents
	# of one version get distinct keys and distinct run titles.
	dispatch_key="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESERVATION}-${cell}"
	# MUST match uat-run.yaml's run-name exactly (now includes intent).
	title="UAT ${RESERVATION} ${intent} @ ${label} #${dispatch_key}"
	echo "::group::${title} [intent=${intent}]"

	gh workflow run uat-run.yaml --repo "$REPO" --ref "$REF" \
	-f reservation="$RESERVATION" -f aicr_version="$ver" \
	-f intent="$intent" \
	-f dispatch_key="$dispatch_key"

	# Resolve the dispatched run by its run-name — this title MUST match
	# uat-run.yaml's `run-name:` (which appends the same dispatch_key).
	# dispatch_key makes the title globally unique, so an exact title
	# match is unambiguous; no createdAt filter (a runner clock running
	# ahead of GitHub's API clock would falsely reject this dispatch).
	run_id=""
	for _ in $(seq 1 "$POLL_MAX_ATTEMPTS"); do
	sleep "$POLL_INTERVAL_SECONDS"
	run_id=$(gh run list --repo "$REPO" --workflow uat-run.yaml \
	--event workflow_dispatch --limit 50 \
	--json databaseId,displayTitle,createdAt 2>/dev/null \
	\| jq -r --arg t "$title" \
	'map(select(.displayTitle == $t))
	\| sort_by(.createdAt) \| .[-1].databaseId // empty' \|\| true)
	[[ -n "$run_id" ]] && break
	done
	if [[ -z "$run_id" ]]; then
	# Fail the leg rather than continue: an unwatched run can still
	# hold the reservation's pending slot and let a later (lower-
	# priority) cell supersede it, breaking version ordering.
	echo "::error::Dispatched ${title} but could not resolve its run id; stopping this reservation leg to preserve version ordering."
	echo "::endgroup::"
	exit 1
	fi

	echo "Waiting on run ${run_id} ..."
	# --exit-status is intentionally not fatal here; classify the run's
	# conclusion below so we can distinguish a benign superseded run from
	# a real failure without aborting mid-loop.
	# Output is discarded: in a non-TTY log `gh run watch` APPENDS a full
	# status render every refresh (no in-place redraw), which flooded the
	# drive log with tens of thousands of duplicate lines per multi-hour
	# cell. We only need it to BLOCK until the run finishes; the
	# conclusion is read from `gh run view` below, so the live render is
	# redundant. --interval 60 keeps the polling gentle on the API.
	gh run watch "$run_id" --repo "$REPO" --interval 60 --exit-status >/dev/null 2>&1 \|\| true

	conclusion=$(gh run view "$run_id" --repo "$REPO" --json conclusion --jq '.conclusion' 2>/dev/null \|\| echo "")
	njobs=$(gh api "repos/${REPO}/actions/runs/${run_id}/jobs" --jq '.total_count' 2>/dev/null \|\| echo "")
	if [[ "$conclusion" == "cancelled" && "$njobs" == "0" ]]; then
	# Benign: the single-slot reservation lease dropped a pending run.
	# Surface it (must not be silent) but do not fail the leg — the
	# superseded-run observer complements this (DC1 superseded surfacing).
	echo "::warning::${title} [intent=${intent}] was superseded while pending (dropped by the reservation lease); re-dispatch when the reservation frees."
	elif [[ "$conclusion" != "success" ]]; then
	# Real cell failure/timeout — record it so the leg (and thus the
	# nightly batch) fails, but keep testing the remaining cells.
	echo "::error::${title} [intent=${intent}] finished with conclusion '${conclusion:-unknown}'."
	leg_failed=1
	fi
	echo "::endgroup::"
	done
	done

	if [[ -n "$leg_failed" ]]; then
	echo "::error::One or more version cells failed for ${RESERVATION}."
	exit 1
	fi

Navigation Menu

UAT Nightly Batch #42

Workflow file

UAT Nightly Batch #42

Uh oh!

Workflow file for this run