Skip to content

Commit 7224fa4

Browse files
committed
ci(uat): remove AWS GB200 nightly UAT reservation entry
GB200 hardware is being retired. Drop the aws-gb200 row from the UAT reservation registry so the nightly broker no longer dispatches it, and update TestCommittedRegistryValid to match the reduced set (it now locks the exact reservation set so a re-add fails closed). Also scrub the retired Capacity Block CR ID from the GB200 cluster-config (target: "" + note), since the reservation no longer exists. Scoped intentionally: the GB200 cluster/test configs, the janitor job, and the multi-arch build machinery in uat-aws.yaml are left in place for a successor ARM instance type. Signed-off-by: Nathan Hensley <nhensley@nvidia.com>
1 parent 534a9f5 commit 7224fa4

3 files changed

Lines changed: 19 additions & 61 deletions

File tree

infra/uat/reservations.yaml

Lines changed: 0 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -147,51 +147,6 @@ reservations:
147147
nightly-intent-min-versions:
148148
inference: v0.18.0
149149
training: v0.18.0
150-
- name: aws-gb200
151-
slug: ag2
152-
cloud: aws
153-
# CROSS-ACCOUNT: the GB200 Capacity Block lives in a SEPARATE tenancy
154-
# (dgx-cloud-runai-prod-22, account 897722687756), not the standard UAT
155-
# account. The account is NOT a field here: the AWS pipeline derives the
156-
# tenancy it assumes the OIDC role into from the target cluster-config's
157-
# .deployment.tenancy (cluster-config-gb200.yaml sets it to 897722687756),
158-
# the single source of truth. Prereq: apply infra/uat-aws-account in
159-
# 897722687756 (GitHub OIDC provider + github-actions-role-aicr).
160-
# GB200 reservations are Capacity Blocks (marketType: capacity-block in
161-
# cluster-config-gb200.yaml), not standing on-demand CRs like the h100 rows,
162-
# so a run must fall inside the block's active window or actuator apply
163-
# fails to acquire capacity. This particular block (cr-0e2f3833a602809a6,
164-
# CapacityBlockId cbr-0a6a5543dd5c74727) is booked long: State=active,
165-
# StartDate 2025-11-15, EndDate 2029-08-22 (TotalInstanceCount 18). A 04:00
166-
# UTC nightly cron therefore always falls inside the window today, which is
167-
# why nightly enrollment below is safe without broker window-awareness. If
168-
# the block is ever re-booked to a shorter window, add a scheduling gate to
169-
# the broker (or revert to `nightly-intents: []`) before it lapses.
170-
reservation-id: cr-0e2f3833a602809a6
171-
accelerator: gb200
172-
# 4 Blackwell GPUs per p6e-gb200.36xlarge node; the cluster-config runs 2
173-
# nodes (activates the multi-node NCCL East-West fabric check).
174-
gpu-count: 4
175-
cluster-config-path: tests/uat/aws/cluster-config-gb200.yaml
176-
test-config-dir: tests/uat/aws/tests
177-
# Enrolled in the nightly batch with both intents, matching the aws-h100
178-
# row. Training runs the Kubeflow TrainJob CUJ; inference stands up and
179-
# validates the served inference stack (the vLLM serve run itself stays
180-
# disabled repo-wide, #1644). Each (version × intent) is a full
181-
# provision→CUJ→teardown cell dispatched through the shared per-reservation
182-
# lease, so the two intents serialize with no contention. Safe to enroll
183-
# because the Capacity Block is booked active through 2029 (see the block
184-
# note above) and the GB200 EKS UAT config first shipped in v0.18.0.
185-
nightly-intents: [training, inference]
186-
# Gate release cells to v0.18.0: cluster-config-gb200.yaml and the GB200 EKS
187-
# UAT lane first shipped in v0.18.0 (absent in v0.17.0 and earlier), so a
188-
# release cell below that tag has no GB200 config to run and would fail
189-
# closed. `main` always runs both intents (built from source); older
190-
# releases are skipped until v0.18.0 is the oldest cell in the window. Bump
191-
# if the actual first-supported tag differs.
192-
nightly-intent-min-versions:
193-
training: v0.18.0
194-
inference: v0.18.0
195150
- name: kind-h100
196151
slug: kh1
197152
cloud: kind

pkg/uatbroker/registry_test.go

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -809,15 +809,14 @@ func TestDaytimeAssignmentsNone(t *testing.T) {
809809
}
810810

811811
// TestCommittedRegistryValid guards the actual checked-in registry: it must
812-
// parse, validate, and carry the launch reservations plus the aws-gb200
813-
// bring-up row. A bad data edit fails here before it can break the broker
814-
// workflows.
812+
// parse, validate, and carry the launch reservations. A bad data edit fails
813+
// here before it can break the broker workflows.
815814
func TestCommittedRegistryValid(t *testing.T) {
816815
reg, err := LoadRegistryFile(filepath.Join("..", "..", "infra", "uat", "reservations.yaml"))
817816
if err != nil {
818817
t.Fatalf("committed reservations.yaml invalid: %v", err)
819818
}
820-
want := map[string]string{"aws-h100": CloudAWS, "gcp-h100": CloudGCP, "azure-h100": CloudAzure, "aws-gb200": CloudAWS, "kind-h100": CloudKind}
819+
want := map[string]string{"aws-h100": CloudAWS, "gcp-h100": CloudGCP, "azure-h100": CloudAzure, "kind-h100": CloudKind}
821820
for name, cloud := range want {
822821
res, err := reg.Lookup(name)
823822
if err != nil {
@@ -829,6 +828,17 @@ func TestCommittedRegistryValid(t *testing.T) {
829828
}
830829
}
831830

831+
// The registry must carry EXACTLY these reservations. The positive lookup
832+
// loop above would still pass if the retired aws-gb200 row (or any other)
833+
// were re-added, so lock the full set here — a re-add or a stray row fails
834+
// closed rather than sliding in unnoticed.
835+
gotNames := append([]string(nil), reg.Names()...)
836+
slices.Sort(gotNames)
837+
wantNames := []string{"aws-h100", "azure-h100", "gcp-h100", "kind-h100"}
838+
if !slices.Equal(gotNames, wantNames) {
839+
t.Errorf("committed registry reservations = %v, want exactly %v", gotNames, wantNames)
840+
}
841+
832842
// The committed daytime-name discovery slugs (ADR-017). Locked here so a
833843
// future edit cannot silently rename or collide a slug — the daytime
834844
// cluster name (aicr-uat-day-<slug>-<slot>-<run_id>) and its guard/teardown
@@ -837,7 +847,6 @@ func TestCommittedRegistryValid(t *testing.T) {
837847
"aws-h100": "ah1",
838848
"gcp-h100": "gh1",
839849
"azure-h100": "zh1",
840-
"aws-gb200": "ag2",
841850
"kind-h100": "kh1",
842851
}
843852
for name, slug := range wantSlug {
@@ -890,13 +899,6 @@ func TestCommittedRegistryValid(t *testing.T) {
890899
// acceptance run (29125390442); inference joined after a green
891900
// manual intent=inference dispatch.
892901
"azure-h100": {IntentTraining, IntentInference},
893-
// aws-gb200 enrolled with both intents once the GB200 EKS UAT lane
894-
// shipped (v0.18.0) and the Capacity Block was confirmed booked active
895-
// through 2029 (cr-0e2f3833a602809a6). Release cells are gated to
896-
// v0.18.0 via nightly-intent-min-versions (the config first shipped
897-
// there), so only `main` runs GB200 nightly until v0.18.0 ages into the
898-
// window.
899-
"aws-gb200": {IntentTraining, IntentInference},
900902
// kind-h100 (nvkind real-silicon lane, DC5 #1278) enrolled with both
901903
// intents after green manual H100 acceptance runs (training 29954092703,
902904
// inference 29965464868). Release cells are gated to v0.18.0 via
@@ -920,12 +922,10 @@ func TestCommittedRegistryValid(t *testing.T) {
920922
// a future edit cannot silently drop a gate — which would let a release cell
921923
// run a pre-fix aicr and emit failing/unusable evidence. azure-h100 gates its
922924
// AKS perf + driver-only fixes; kind-h100 gates the uat-kind lane + the
923-
// os-agnostic coordinate fix (#1851); aws-gb200 gates the GB200 EKS UAT
924-
// config, which first shipped in v0.18.0. All landed post-v0.17.0.
925+
// os-agnostic coordinate fix (#1851). All landed post-v0.17.0.
925926
wantMinVersions := map[string]map[string]string{
926927
"azure-h100": {IntentTraining: "v0.18.0", IntentInference: "v0.18.0"},
927928
"kind-h100": {IntentTraining: "v0.18.0", IntentInference: "v0.18.0"},
928-
"aws-gb200": {IntentTraining: "v0.18.0", IntentInference: "v0.18.0"},
929929
}
930930
for name, want := range wantMinVersions {
931931
res, lookupErr := reg.Lookup(name)

tests/uat/aws/cluster-config-gb200.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,10 @@ compute:
146146
desired: 2
147147
reservation:
148148
preference: capacity-reservations-only
149-
target: cr-0e2f3833a602809a6
149+
# Capacity Block CR ID removed: the GB200 block is retired and this
150+
# lane is no longer in the reservation registry. Re-fill with the
151+
# successor reservation's id before re-enabling.
152+
target: ""
150153
marketType: capacity-block # REQUIRED — GB200 CRs are Capacity Blocks
151154
labels:
152155
nodeGroup: gpu-worker

0 commit comments

Comments
 (0)