UAT Daytime Human-Access #66
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: UAT Daytime Human-Access | |
| # The day side of the day/night cycle (#1281, DC8): stand up ONE long-lived, | |
| # human-facing deployment per cloud ON DEMAND (manual daytime-up) for the working | |
| # day, with an automatic evening teardown that releases the reservation before the | |
| # nightly batch. daytime-up is intentionally NOT on a cron — a human requests it | |
| # when they want a cluster (see docs/contributor/uat.md); only the safety-net | |
| # teardown is scheduled, so a manually-provisioned daytime cluster is reclaimed | |
| # even if nobody remembers to tear it down. If that teardown itself fails, the | |
| # pre-batch guard blocks the batch rather than racing the still-held cluster. The cloud→flavor split is data, | |
| # not code — it comes from the `daytime-intent` column of the reservation registry | |
| # (infra/uat/reservations.yaml), defaulting to AWS=training / GCP=inference. | |
| # | |
| # This workflow is a THIN SCHEDULER. It owns no lifecycle mechanics: it enumerates | |
| # the daytime rotation from the broker and dispatches the shared uat-run.yaml | |
| # once per reservation with lifecycle=daytime-up (manual) or daytime-down (manual | |
| # or the evening cron). DC2 owns provision-and-hold, teardown, and the pre-batch | |
| # guard; uat-run.yaml owns the per-reservation lease. Because the daytime runs go | |
| # through that same dispatch surface, they contend on the SAME lease as the nightly | |
| # batch — CI and human use never overlap on one reservation. | |
| # | |
| # The daytime cluster is NOT a UAT cell: daytime-up stops after deploy (no CUJ), | |
| # so it emits NO evidence bundle and produces NO TestGrid column. Access is | |
| # distributed OUT-OF-BAND (see docs/contributor/uat.md), never through this CI | |
| # path. | |
| on: | |
| # daytime-UP is manual only (workflow_dispatch, action=up). The single scheduled | |
| # edge is the evening DOWN — an unconditional safety-net teardown. | |
| schedule: | |
| # Times are UTC. The nightly batch opens at 04:00 UTC (uat-nightly-batch.yaml). | |
| # KEEP IN LOCKSTEP with env.EVENING_DOWN_CRON below — the Resolve-action step | |
| # maps `github.event.schedule` to `down` by an exact string match against that | |
| # env value (the `on:` block cannot reference env, so this coupling is by | |
| # convention). A cron with no matching env value fails with | |
| # `::error::could not resolve daytime action` when it fires. | |
| - cron: '0 2 * * *' # evening teardown — release the reservation ~2h before the batch (daytime-down) | |
| workflow_dispatch: | |
| inputs: | |
| action: | |
| description: 'up: provision + deploy the daytime cluster; holds on success, tears down on failure or cancellation. down: tear it down and release the reservation.' | |
| type: choice | |
| options: [up, down] | |
| required: true | |
| permissions: | |
| contents: read | |
| # Serialize daytime scheduler runs so a manual dispatch does not fan out a | |
| # second up/down alongside a cron one. (uat-run.yaml's per-reservation lease | |
| # would queue any duplicates anyway; there is no reason to start them.) | |
| concurrency: | |
| group: uat-daytime | |
| cancel-in-progress: false | |
| # The evening cron edge maps to the daytime-down action; a manual dispatch names | |
| # the action (up or down) directly. Consumed by the enumerate job's Resolve action | |
| # step, which maps `github.event.schedule` to `down` by an EXACT string match | |
| # against this value — which MUST stay identical to the on.schedule cron above (an | |
| # unsynced edit fails loud, but only when it fires). | |
| env: | |
| EVENING_DOWN_CRON: '0 2 * * *' # must equal the daytime-down cron in on.schedule | |
| jobs: | |
| # Resolve the up/down action for this trigger and enumerate the daytime | |
| # rotation (reservation + intent) from the registry into a JSON matrix. Keeps | |
| # the scheduler data-driven: onboarding a daytime reservation is a | |
| # `daytime-intent:` edit in infra/uat/reservations.yaml, no workflow change. | |
| enumerate: | |
| if: github.repository == 'nvidia/aicr' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| action: ${{ steps.action.outputs.action }} | |
| lifecycle: ${{ steps.action.outputs.lifecycle }} | |
| matrix: ${{ steps.list.outputs.matrix }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Resolve action | |
| id: action | |
| env: | |
| # github.event.schedule is the cron string that fired (empty on | |
| # workflow_dispatch); inputs.action is set only on workflow_dispatch. | |
| EVENT_NAME: ${{ github.event_name }} | |
| SCHEDULE: ${{ github.event.schedule }} | |
| INPUT_ACTION: ${{ inputs.action }} | |
| run: | | |
| set -euo pipefail | |
| if [[ "${EVENT_NAME}" == "workflow_dispatch" ]]; then | |
| action="${INPUT_ACTION}" | |
| elif [[ "${SCHEDULE}" == "${EVENING_DOWN_CRON}" ]]; then | |
| action="down" | |
| else | |
| echo "::error::could not resolve daytime action (event='${EVENT_NAME}', schedule='${SCHEDULE}')" | |
| exit 1 | |
| fi | |
| case "${action}" in | |
| up|down) ;; | |
| *) echo "::error::invalid action '${action}' (want up|down)"; exit 1 ;; | |
| esac | |
| echo "action=${action}" >> "$GITHUB_OUTPUT" | |
| echo "lifecycle=daytime-${action}" >> "$GITHUB_OUTPUT" | |
| echo "Daytime action: ${action} (lifecycle=daytime-${action})" | |
| - name: Load versions | |
| id: versions | |
| uses: ./.github/actions/load-versions | |
| - name: Setup Go | |
| uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 | |
| with: | |
| go-version: '${{ steps.versions.outputs.go }}' | |
| cache: true | |
| cache-dependency-path: | | |
| go.sum | |
| vendor/modules.txt | |
| - name: Enumerate daytime rotation | |
| id: list | |
| env: | |
| GOFLAGS: -mod=vendor | |
| ACTION: ${{ steps.action.outputs.action }} | |
| run: | | |
| set -euo pipefail | |
| go build -o ./bin/uat-broker ./tools/uat-broker | |
| # Compact to one line for GITHUB_OUTPUT; the matrix is a JSON array of | |
| # {reservation,intent} objects consumed as strategy.matrix.include. | |
| matrix=$(./bin/uat-broker reservations --daytime | jq -c .) | |
| echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" | |
| echo "Daytime rotation: ${matrix}" | |
| if [[ "${matrix}" == "[]" ]]; then | |
| echo "::notice::no reservations carry a daytime-intent; nothing to ${ACTION}." | |
| fi | |
| # One matrix leg per daytime reservation (parallel — independent hardware). | |
| # Each leg DISPATCHES uat-run.yaml once (workflow_dispatch is exempt from the | |
| # GITHUB_TOKEN recursion rule and always creates a run) and watches it to | |
| # completion so a failed morning handoff / evening teardown surfaces here. The | |
| # dispatched uat-run.yaml runs top-level with its own permissions and holds the | |
| # reservation lease; this job only needs actions:write to dispatch + watch. | |
| drive: | |
| needs: enumerate | |
| if: needs.enumerate.outputs.matrix != '[]' | |
| runs-on: ubuntu-latest | |
| # A daytime-up dispatch (provision + deploy) can run ~90 min; watching it | |
| # blocks this job for that long. Stay under GitHub's 6h hosted-job cap. | |
| timeout-minutes: 200 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJSON(needs.enumerate.outputs.matrix) }} | |
| permissions: | |
| contents: read # checkout | |
| actions: write # dispatch uat-run.yaml and watch the dispatched run | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Dispatch and watch the daytime run | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| REPO: ${{ github.repository }} | |
| REF: ${{ github.ref_name }} | |
| RESERVATION: ${{ matrix.reservation }} | |
| INTENT: ${{ matrix.intent }} | |
| ACTION: ${{ needs.enumerate.outputs.action }} | |
| LIFECYCLE: ${{ needs.enumerate.outputs.lifecycle }} | |
| run: | | |
| set -euo pipefail | |
| # Poll budget for finding the dispatched run (gh workflow run returns | |
| # no run id): POLL_MAX_ATTEMPTS x POLL_INTERVAL_SECONDS seconds. | |
| POLL_MAX_ATTEMPTS=24 | |
| POLL_INTERVAL_SECONDS=5 | |
| # Unique per-dispatch key (this run's id + attempt + reservation + | |
| # action) so the resolver below watches THIS run, never a concurrent | |
| # manual dispatch of the same reservation. Mirrors the nightly | |
| # controller's correlation scheme. | |
| dispatch_key="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESERVATION}-${ACTION}" | |
| # aicr_version is left empty (daytime always runs main/tip), so | |
| # uat-run.yaml's run-name renders '@ main'; the dispatch_key suffix | |
| # makes the title globally unique. MUST match uat-run.yaml's run-name | |
| # exactly (now includes intent). | |
| title="UAT ${RESERVATION} ${INTENT} @ main #${dispatch_key}" | |
| echo "::group::${title}" | |
| gh workflow run uat-run.yaml --repo "$REPO" --ref "$REF" \ | |
| -f reservation="$RESERVATION" \ | |
| -f intent="$INTENT" \ | |
| -f lifecycle="$LIFECYCLE" \ | |
| -f dispatch_key="$dispatch_key" | |
| # Resolve the dispatched run by its (globally unique) run-name; no | |
| # createdAt filter, so a runner clock ahead of GitHub's API clock | |
| # cannot falsely reject this dispatch. | |
| run_id="" | |
| for _ in $(seq 1 "$POLL_MAX_ATTEMPTS"); do | |
| sleep "$POLL_INTERVAL_SECONDS" | |
| run_id=$(gh run list --repo "$REPO" --workflow uat-run.yaml \ | |
| --event workflow_dispatch --limit 50 \ | |
| --json databaseId,displayTitle,createdAt 2>/dev/null \ | |
| | jq -r --arg t "$title" \ | |
| 'map(select(.displayTitle == $t)) | |
| | sort_by(.createdAt) | .[-1].databaseId // empty' || true) | |
| [[ -n "$run_id" ]] && break | |
| done | |
| if [[ -z "$run_id" ]]; then | |
| echo "::error::Dispatched ${title} but could not resolve its run id." | |
| echo "::endgroup::" | |
| exit 1 | |
| fi | |
| echo "Waiting on run ${run_id} ..." | |
| # --exit-status is intentionally not fatal here; classify the run's | |
| # conclusion below so a benign supersede is not mistaken for a failure. | |
| gh run watch "$run_id" --repo "$REPO" --exit-status || true | |
| conclusion=$(gh run view "$run_id" --repo "$REPO" --json conclusion --jq '.conclusion' 2>/dev/null || echo "") | |
| njobs=$(gh api "repos/${REPO}/actions/runs/${run_id}/jobs" --jq '.total_count' 2>/dev/null || echo "") | |
| echo "::endgroup::" | |
| if [[ "$conclusion" == "cancelled" && "$njobs" == "0" ]]; then | |
| # Benign: the single-slot reservation lease dropped this run while it | |
| # was still pending (the nightly batch or another daytime run held | |
| # the reservation). Surface it — must not be silent — but do not fail: | |
| # re-dispatch when the reservation frees. | |
| echo "::warning::${title} was superseded while pending (dropped by the reservation lease); re-run when the reservation frees." | |
| elif [[ "$conclusion" != "success" ]]; then | |
| echo "::error::daytime ${ACTION} for ${RESERVATION} finished with conclusion '${conclusion:-unknown}'." | |
| exit 1 | |
| fi | |
| echo "daytime ${ACTION} for ${RESERVATION} (${INTENT}): ${conclusion}" | |
| - name: Summary | |
| if: always() | |
| env: | |
| SUMMARY_RESERVATION: ${{ matrix.reservation }} | |
| SUMMARY_INTENT: ${{ matrix.intent }} | |
| SUMMARY_ACTION: ${{ needs.enumerate.outputs.action }} | |
| run: | | |
| { | |
| echo "## Daytime human-access — ${SUMMARY_ACTION}" | |
| echo "" | |
| printf '**Reservation:** `%s` · **Intent:** `%s` · **Action:** `%s`\n' \ | |
| "$SUMMARY_RESERVATION" "$SUMMARY_INTENT" "$SUMMARY_ACTION" | |
| echo "" | |
| echo "Access is distributed out-of-band; this cluster emits no evidence bundle and no TestGrid column." | |
| } >> "$GITHUB_STEP_SUMMARY" |