Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 0 additions & 45 deletions infra/uat/reservations.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -147,51 +147,6 @@ reservations:
nightly-intent-min-versions:
inference: v0.18.0
training: v0.18.0
- name: aws-gb200
slug: ag2
cloud: aws
# CROSS-ACCOUNT: the GB200 Capacity Block lives in a SEPARATE tenancy
# (dgx-cloud-runai-prod-22, account 897722687756), not the standard UAT
# account. The account is NOT a field here: the AWS pipeline derives the
# tenancy it assumes the OIDC role into from the target cluster-config's
# .deployment.tenancy (cluster-config-gb200.yaml sets it to 897722687756),
# the single source of truth. Prereq: apply infra/uat-aws-account in
# 897722687756 (GitHub OIDC provider + github-actions-role-aicr).
# GB200 reservations are Capacity Blocks (marketType: capacity-block in
# cluster-config-gb200.yaml), not standing on-demand CRs like the h100 rows,
# so a run must fall inside the block's active window or actuator apply
# fails to acquire capacity. This particular block (cr-0e2f3833a602809a6,
# CapacityBlockId cbr-0a6a5543dd5c74727) is booked long: State=active,
# StartDate 2025-11-15, EndDate 2029-08-22 (TotalInstanceCount 18). A 04:00
# UTC nightly cron therefore always falls inside the window today, which is
# why nightly enrollment below is safe without broker window-awareness. If
# the block is ever re-booked to a shorter window, add a scheduling gate to
# the broker (or revert to `nightly-intents: []`) before it lapses.
reservation-id: cr-0e2f3833a602809a6
accelerator: gb200
# 4 Blackwell GPUs per p6e-gb200.36xlarge node; the cluster-config runs 2
# nodes (activates the multi-node NCCL East-West fabric check).
gpu-count: 4
cluster-config-path: tests/uat/aws/cluster-config-gb200.yaml
test-config-dir: tests/uat/aws/tests
# Enrolled in the nightly batch with both intents, matching the aws-h100
# row. Training runs the Kubeflow TrainJob CUJ; inference stands up and
# validates the served inference stack (the vLLM serve run itself stays
# disabled repo-wide, #1644). Each (version × intent) is a full
# provision→CUJ→teardown cell dispatched through the shared per-reservation
# lease, so the two intents serialize with no contention. Safe to enroll
# because the Capacity Block is booked active through 2029 (see the block
# note above) and the GB200 EKS UAT config first shipped in v0.18.0.
nightly-intents: [training, inference]
# Gate release cells to v0.18.0: cluster-config-gb200.yaml and the GB200 EKS
# UAT lane first shipped in v0.18.0 (absent in v0.17.0 and earlier), so a
# release cell below that tag has no GB200 config to run and would fail
# closed. `main` always runs both intents (built from source); older
# releases are skipped until v0.18.0 is the oldest cell in the window. Bump
# if the actual first-supported tag differs.
nightly-intent-min-versions:
training: v0.18.0
inference: v0.18.0
- name: kind-h100
slug: kh1
cloud: kind
Expand Down
30 changes: 15 additions & 15 deletions pkg/uatbroker/registry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -809,15 +809,14 @@ func TestDaytimeAssignmentsNone(t *testing.T) {
}

// TestCommittedRegistryValid guards the actual checked-in registry: it must
// parse, validate, and carry the launch reservations plus the aws-gb200
// bring-up row. A bad data edit fails here before it can break the broker
// workflows.
// parse, validate, and carry the launch reservations. A bad data edit fails
// here before it can break the broker workflows.
func TestCommittedRegistryValid(t *testing.T) {
reg, err := LoadRegistryFile(filepath.Join("..", "..", "infra", "uat", "reservations.yaml"))
if err != nil {
t.Fatalf("committed reservations.yaml invalid: %v", err)
}
want := map[string]string{"aws-h100": CloudAWS, "gcp-h100": CloudGCP, "azure-h100": CloudAzure, "aws-gb200": CloudAWS, "kind-h100": CloudKind}
want := map[string]string{"aws-h100": CloudAWS, "gcp-h100": CloudGCP, "azure-h100": CloudAzure, "kind-h100": CloudKind}
for name, cloud := range want {
res, err := reg.Lookup(name)
if err != nil {
Expand All @@ -829,6 +828,17 @@ func TestCommittedRegistryValid(t *testing.T) {
}
}

// The registry must carry EXACTLY these reservations. The positive lookup
// loop above would still pass if the retired aws-gb200 row (or any other)
// were re-added, so lock the full set here — a re-add or a stray row fails
// closed rather than sliding in unnoticed.
gotNames := append([]string(nil), reg.Names()...)
slices.Sort(gotNames)
wantNames := []string{"aws-h100", "azure-h100", "gcp-h100", "kind-h100"}
if !slices.Equal(gotNames, wantNames) {
t.Errorf("committed registry reservations = %v, want exactly %v", gotNames, wantNames)
}

// The committed daytime-name discovery slugs (ADR-017). Locked here so a
// future edit cannot silently rename or collide a slug — the daytime
// cluster name (aicr-uat-day-<slug>-<slot>-<run_id>) and its guard/teardown
Expand All @@ -837,7 +847,6 @@ func TestCommittedRegistryValid(t *testing.T) {
"aws-h100": "ah1",
"gcp-h100": "gh1",
"azure-h100": "zh1",
"aws-gb200": "ag2",
"kind-h100": "kh1",
}
for name, slug := range wantSlug {
Expand Down Expand Up @@ -890,13 +899,6 @@ func TestCommittedRegistryValid(t *testing.T) {
// acceptance run (29125390442); inference joined after a green
// manual intent=inference dispatch.
"azure-h100": {IntentTraining, IntentInference},
// aws-gb200 enrolled with both intents once the GB200 EKS UAT lane
// shipped (v0.18.0) and the Capacity Block was confirmed booked active
// through 2029 (cr-0e2f3833a602809a6). Release cells are gated to
// v0.18.0 via nightly-intent-min-versions (the config first shipped
// there), so only `main` runs GB200 nightly until v0.18.0 ages into the
// window.
"aws-gb200": {IntentTraining, IntentInference},
// kind-h100 (nvkind real-silicon lane, DC5 #1278) enrolled with both
// intents after green manual H100 acceptance runs (training 29954092703,
// inference 29965464868). Release cells are gated to v0.18.0 via
Expand All @@ -920,12 +922,10 @@ func TestCommittedRegistryValid(t *testing.T) {
// a future edit cannot silently drop a gate — which would let a release cell
// run a pre-fix aicr and emit failing/unusable evidence. azure-h100 gates its
// AKS perf + driver-only fixes; kind-h100 gates the uat-kind lane + the
// os-agnostic coordinate fix (#1851); aws-gb200 gates the GB200 EKS UAT
// config, which first shipped in v0.18.0. All landed post-v0.17.0.
// os-agnostic coordinate fix (#1851). All landed post-v0.17.0.
wantMinVersions := map[string]map[string]string{
"azure-h100": {IntentTraining: "v0.18.0", IntentInference: "v0.18.0"},
"kind-h100": {IntentTraining: "v0.18.0", IntentInference: "v0.18.0"},
"aws-gb200": {IntentTraining: "v0.18.0", IntentInference: "v0.18.0"},
}
for name, want := range wantMinVersions {
res, lookupErr := reg.Lookup(name)
Expand Down
5 changes: 4 additions & 1 deletion tests/uat/aws/cluster-config-gb200.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,10 @@ compute:
desired: 2
reservation:
preference: capacity-reservations-only
target: cr-0e2f3833a602809a6
# Capacity Block CR ID removed: the GB200 block is retired and this
# lane is no longer in the reservation registry. Re-fill with the
# successor reservation's id before re-enabling.
target: ""
marketType: capacity-block # REQUIRED — GB200 CRs are Capacity Blocks
labels:
nodeGroup: gpu-worker
Expand Down
Loading