6868 description : ' Skip UAT test execution (manual runs only; the nightly batch always tests).'
6969 type : boolean
7070 default : false
71+ deployer :
72+ # helmfile = <accelerator>-<intent>-config.yaml (the existing
73+ # filename shape every cell has always used). argocd = insert
74+ # -argocd before -config.yaml (e.g. h100-training-argocd-config.yaml).
75+ # See issue #2194.
76+ description : ' Deployer variant.'
77+ type : string
78+ default : helmfile
7179
7280permissions :
7381 contents : read
@@ -88,17 +96,28 @@ jobs:
8896 if : always() && github.repository == 'nvidia/aicr'
8997 runs-on : ubuntu-latest
9098 # Budget = uncapped non-UAT steps (build, validator image push, EKS
91- # provisioning, evidence upload ~30m) + UAT phase steps (prep 15 + install 90
92- # + validate 40 + the intent-selected CUJ (train 25 OR serve 40) + verify 5
93- # = up to 190) + ~8 failure-path debug collection (Collect cluster debug
94- # bundle) + the always()-run Destroy Cluster teardown. Only one of
95- # train/serve runs per intent, so the budget counts the larger (serve).
96- # Teardown = up to 3 retries × ~18m real destroy = up to ~54m. The teardown
97- # MUST fit in this budget: a job-level timeout cancels pending always()
98- # steps, so an undersized cap would skip teardown and leak the GPU node /
99- # capacity reservation even after a LATE failure. ~30 setup + 190 UAT + ~8
100- # debug = 228, + ~54 teardown = 282, so 300 leaves the teardown clear
101- # headroom on a worst-case run.
99+ # provisioning, evidence upload ~30m) + UAT phase steps + ~8 failure-path
100+ # debug collection + the always()-run Destroy Cluster teardown (up to 3
101+ # retries × ~18m real destroy = up to ~54m). The teardown MUST fit in this
102+ # cap: a job-level timeout cancels pending always() steps, so an undersized
103+ # cap would skip teardown and leak the GPU node / capacity reservation
104+ # even after a LATE failure. Only one of train/serve runs per intent, so
105+ # the UAT phase-step count uses the larger.
106+ #
107+ # UAT-phase-step budgets by (deployer, intent), using the current
108+ # timeout-minutes on each step below:
109+ # helmfile+training: prep 15 + install 90 + validate 40 + train 25 + verify 5 = 175
110+ # helmfile+inference: prep 15 + install 90 + validate 40 + serve 40 + verify 5 = 190
111+ # argocd+training: prep 15 + install 110 + validate 40 + train 25 + verify 5 = 195 (issue #2194, aws-h100 only)
112+ # argocd+inference: not shipped today; would be 15 + 110 + 40 + 40 + 5 = 210 (needs cap bump below)
113+ #
114+ # Total = ~30 setup + <UAT> + ~8 debug + ~54 teardown. On today's
115+ # worst-case (argocd+training or helmfile+inference), ~30 + max(195,190)
116+ # + ~8 + ~54 = ~287, so 300 leaves ~13m of teardown headroom.
117+ # NOTE: the "not shipped today" argocd+inference row would push total to
118+ # ~302, over the cap. If a future PR wires the argocd variant for the
119+ # inference cell (or re-enables phase_serve alongside argocd), raise this
120+ # to ~330 in the same PR.
102121 timeout-minutes : 300
103122 permissions :
104123 contents : read
@@ -139,10 +158,19 @@ jobs:
139158 && format('aicr-uat-day-{0}-{1}-{2}', inputs.slug, inputs.slot, github.run_id)
140159 || format('aicr-uat-{0}', github.run_id) }}
141160CLUSTER_CONFIG : ${{ inputs.cluster_config_path }}
142- # The AICRConfig is selected by accelerator + intent
143- # (<accelerator>-<intent>-config.yaml); both intents drive the same
144- # cluster-config (GPU pool from the reservation, system/CPU pools dynamic).
145- TEST_CONFIG : ${{ inputs.test_config_dir }}/${{ inputs.accelerator }}-${{ inputs.intent }}-config.yaml
161+ # The AICRConfig is selected by accelerator + intent + deployer:
162+ # deployer=helmfile → <accelerator>-<intent>-config.yaml (the default;
163+ # backwards-compatible with every existing cell)
164+ # deployer=X (X!=helmfile) → <accelerator>-<intent>-X-config.yaml
165+ # (e.g. deployer=argocd → h100-training-argocd-config.yaml, issue #2194)
166+ # Both variants drive the same cluster-config (GPU pool from the
167+ # reservation, system/CPU pools dynamic). The step-level check below asserts
168+ # the resolved file exists so an unwired deployer variant fails early
169+ # (before any provisioning) rather than deep in phase_prep.
170+ TEST_CONFIG : >-
171+ ${{ inputs.test_config_dir }}/${{ inputs.accelerator }}-${{ inputs.intent }}${{
172+ inputs.deployer != 'helmfile' && format('-{0}', inputs.deployer) || ''
173+ }}-config.yaml
146174# v0.4.27 — required for node-group taints in the cluster-config schema
147175 # (GPU pool carries skyhook.nvidia.com=runtime-required:NoSchedule until
148176 # NodeWrite finishes tuning + reboot). Earlier v0.4.x pins predate taint
@@ -190,6 +218,7 @@ jobs:
190218 LIFECYCLE : ${{ inputs.lifecycle }}
191219 SLUG : ${{ inputs.slug }}
192220 SLOT : ${{ inputs.slot }}
221+ DEPLOYER : ${{ inputs.deployer }}
193222 run : |
194223 set -euo pipefail
195224 case "${INTENT}" in
@@ -233,6 +262,15 @@ jobs:
233262 echo "::error::${CLUSTER_CONFIG} has no .deployment.tenancy"; exit 1
234263 fi
235264 echo "AWS_ACCOUNT_ID=${TENANCY}" >> "${GITHUB_ENV}"
265+ # Validate deployer after the account export (which teardown depends on).
266+ # workflow_dispatch constrains this to a choice, but workflow_call passes
267+ # free strings, so a typo from a caller must surface as a clear error here
268+ # rather than as a missing test-config file below or a mysterious dispatch
269+ # deep in phases.sh. Allowlist mirrors uat-run.yaml's choice input.
270+ case "${DEPLOYER}" in
271+ helmfile|argocd) ;;
272+ *) echo "::error::unsupported deployer '${DEPLOYER}' (want helmfile|argocd)"; exit 1 ;;
273+ esac
236274 # The test config only matters for the run phases (nightly / daytime-up
237275 # provision+deploy). daytime-down just tears the held cluster down and
238276 # never reads it, so skip the existence check there — a missing sibling
@@ -685,21 +723,36 @@ jobs:
685723 aws-region : ${{ env.AWS_REGION }}
686724 role-session-name : GitHubActions-UAT-AICR-Install
687725
688- - name : UAT - install (helmfile apply)
726+ - name : UAT - install (helmfile apply or argocd sync )
689727 id : install
690728 if : steps.prep.outcome == 'success'
691- # helmfile apply (up to HELMFILE_TIMEOUT_SECONDS, 20m) + the post-install
692- # readiness gate, which runs `aicr validate --phase deployment` until it
693- # passes READINESS_CONSECUTIVE_PASSES times (up to READINESS_TIMEOUT_SECONDS,
694- # 60m, spanning nodewright tuning + reboots, which can run past 30m on a
695- # cold GPU node). Sized to exceed one helmfile attempt + the full gate
696- # window (20 + 60 = 80, + margin) so the gate's own fail-closed path runs
697- # before GitHub Actions kills the step.
698- timeout-minutes : 90
729+ # Step-level timeout must fit the LARGER of the two deployer branches,
730+ # because both dispatch from `./run install` and share this cap:
731+ # helmfile: HELMFILE_TIMEOUT_SECONDS 20m + gate 60m = 80m
732+ # argocd: helm-install 5m + apply+sync 30m + root-grace 2m + gate 60m = 97m
733+ # ARGOCD_HELM_TIMEOUT_SECONDS + ARGOCD_SYNC_TIMEOUT_SECONDS +
734+ # ARGOCD_ROOT_APP_GRACE_SECONDS + READINESS_TIMEOUT_SECONDS in
735+ # tests/uat/lib/phases.sh drive the argocd side. Sized at 110m so the
736+ # argocd fail-closed paths (each shared-budget retry loop's `::error::`)
737+ # run before GitHub Actions kills the step. The gate's own fail-closed
738+ # path still fires within budget on both branches.
739+ timeout-minutes : 110
699740 shell : bash
700741 env :
701742 AICR_BIN : ${{ github.workspace }}/aicr
702743 RUN_ID : ${{ github.run_id }}
744+ # Consumed only by the argocd deployer branch (install_argocd in
745+ # tests/uat/lib/phases.sh), which provisions an in-cluster ghcr.io
746+ # repo-creds Secret so Argo CD's repo-server can pull the bundle
747+ # pushed by phase_prep. GITHUB_ACTOR is auto-injected by the
748+ # Actions runtime; GITHUB_TOKEN is not, so pass it explicitly.
749+ # The token is scoped to `packages: write` at the job level (see
750+ # permissions block above), which is exactly what the Secret
751+ # needs. Set the token ONLY for the argocd branch so helmfile's
752+ # kubectl/aicr/chart-hook children don't inherit it in-env;
753+ # install_argocd's `:?` guard fires on the empty default, which
754+ # is what we want if a caller misconfigures the deployer input.
755+ GITHUB_TOKEN : ${{ inputs.deployer == 'argocd' && github.token || '' }}
703756 run : ./tests/uat/aws/run install "${TEST_CONFIG}"
704757
705758 # Runs ALL validate phases (deployment + conformance + performance), not
@@ -883,12 +936,20 @@ jobs:
883936 SUMMARY_AICR_VERSION : ${{ inputs.aicr_version }}
884937 SUMMARY_INTENT : ${{ inputs.intent }}
885938 SUMMARY_LIFECYCLE : ${{ inputs.lifecycle }}
939+ SUMMARY_DEPLOYER : ${{ inputs.deployer }}
886940 run : |
887941 {
888942 echo "## UAT Results (AWS)"
889943 echo ""
890944 printf '**Reservation:** `%s` · **Intent:** `%s` · **Lifecycle:** `%s`\n' \
891945 "$SUMMARY_RESERVATION" "$SUMMARY_INTENT" "$SUMMARY_LIFECYCLE"
946+ # Render the deployer only when it deviates from the default
947+ # (helmfile), so existing cells' summaries are unchanged and
948+ # only argocd cells surface the extra line — makes it easy to
949+ # spot which lane a run was in from the Test Summary alone.
950+ if [[ "$SUMMARY_DEPLOYER" != "" && "$SUMMARY_DEPLOYER" != "helmfile" ]]; then
951+ printf '**Deployer:** `%s`\n' "$SUMMARY_DEPLOYER"
952+ fi
892953 printf '**Cluster:** `%s`\n' "$DEPLOYMENT_ID"
893954 printf '**AICR version:** `%s`\n' "${SUMMARY_AICR_VERSION:-main (build from source)}"
894955 echo "**Build:** \`${{ github.sha }}\` (branch: \`${{ github.ref_name }}\`)"
0 commit comments