UAT Janitor #372
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: UAT Janitor | |
| # Backstop teardown for orphaned UAT clusters + resource groups across all three | |
| # clouds. The in-run "Destroy Cluster" step (uat-{gcp,aws,azure}.yaml) is the | |
| # first line; it cannot cover every case (a runner hard-killed mid-cancel, a | |
| # daytime held cluster whose evening daytime-down never fired, an abandoned | |
| # skip_delete run). This reaps whatever the in-run teardown missed, reconciling | |
| # each aicr-uat-<run_id> deployment against its owning GitHub run. See | |
| # .github/scripts/uat-janitor.sh for the full safety model. | |
| # | |
| # ROLLOUT: dry-run-first. The scheduled run always reports only (DRY_RUN=true) — | |
| # it never deletes. To actually reap, dispatch this workflow with enforce=true | |
| # (a deliberate human action). Once the scheduled dry-run logs have been trusted | |
| # over a few cycles, flip the schedule to enforce by changing the DRY_RUN env | |
| # expression below to default false for the schedule event. | |
| # | |
| # PREREQUISITE (verify before enforce): each cloud's OIDC/WIF trust must permit | |
| # this workflow. The federated identities are subject-pinned to NVIDIA/aicr refs | |
| # (main + test/uat). Every job is additionally gated on | |
| # github.ref == 'refs/heads/main', so a workflow_dispatch that selects another | |
| # branch (the GCP/Azure trusts also admit test/uat) cannot run that branch's copy | |
| # of the janitor with delete authority — scheduled + dispatch runs only ever | |
| # execute main's version. A ref-scoped trust already covers this; if any trust is | |
| # workflow-file-scoped, add uat-janitor.yaml. The first scheduled dry-run surfaces | |
| # any auth gap harmlessly. | |
| on: | |
| schedule: | |
| - cron: '17 * * * *' # hourly, offset from top-of-hour job contention | |
| workflow_dispatch: | |
| inputs: | |
| cloud: | |
| description: 'Which cloud(s) to reconcile.' | |
| type: choice | |
| options: [all, gcp, aws, azure] | |
| default: all | |
| enforce: | |
| description: 'Actually delete orphans (default: dry-run report only).' | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: read | |
| # Never let two janitor runs reconcile the same cloud state concurrently; let an | |
| # in-flight reap finish rather than cancelling it mid-destroy. | |
| # | |
| # Deliberately NOT the per-reservation `uat-<reservation>` group that uat-run.yaml | |
| # leases. That would be a cross-workflow mutex, and the exclusion it buys is | |
| # already provided by the design: | |
| # * Janitor candidates are RUN-SCOPED unique deployment ids. A live lifecycle | |
| # run reports `queued`/`in_progress` on the GitHub API, so classify() skips | |
| # it outright — its cluster is never a candidate in the first place. | |
| # * A lifecycle run touches only its OWN run-scoped name and Terraform state | |
| # prefix, so it shares no resource with whatever older orphan the janitor is | |
| # targeting. There is no collision to serialize. | |
| # * Actuator destroys are idempotent, and GCP/Azure additionally serialize on | |
| # the Terraform state lock. | |
| # * Daytime last-mile: a daytime name encodes the daytime-UP run id, so the | |
| # run-status liveness gate cannot see an active daytime-DOWN teardown of the | |
| # same cluster (a different run). The teardown always executes as a | |
| # uat-run.yaml run — the evening scheduler dispatches uat-run.yaml as its own | |
| # top-level run, and a direct `gh workflow run uat-run.yaml -f | |
| # lifecycle=daytime-down` is a uat-run.yaml run too — so classify() calls | |
| # uat_lifecycle_active() and, if ANY uat-run run is in flight, defers | |
| # (SKIP:lifecycle-active) to the next hourly cycle. This is a liveness PROBE, | |
| # not a mutex: it collapses the common race (a down already running) to near | |
| # zero but leaves a residual TOCTOU window — a down that STARTS in the ~1h | |
| # between the check and the reap. The daytime>=24h floor and the DRY_RUN | |
| # default backstop that window (break_state_lock is best-effort and idempotent; | |
| # an AWS destroy has no lock but the actuator destroy is itself idempotent). | |
| # For a hard "never races" guarantee a shared cross-workflow lease would be | |
| # required; that is the tracked follow-up (#2132) to revisit before scheduled | |
| # enforcement is enabled. With DRY_RUN defaulting true, the liveness probe, and the | |
| # age floors, the residual window does not justify that coupling for this backstop | |
| # today. | |
| concurrency: | |
| group: uat-janitor | |
| cancel-in-progress: false | |
| env: | |
| # Scheduled runs are always dry-run; only an explicit enforce=true dispatch | |
| # deletes. inputs.enforce is null for the schedule event, so this is 'true' there. | |
| DRY_RUN: ${{ inputs.enforce == true && 'false' || 'true' }} | |
| jobs: | |
| gcp: | |
| if: >- | |
| github.repository == 'nvidia/aicr' && | |
| github.ref == 'refs/heads/main' && | |
| (github.event_name == 'schedule' || inputs.cloud == 'all' || inputs.cloud == 'gcp') | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| permissions: | |
| contents: read | |
| id-token: write | |
| actions: read | |
| env: | |
| GCP_PROJECT_ID: "eidosx" | |
| JANITOR_CONFIG: "tests/uat/gcp/cluster-config.yaml" | |
| JANITOR_ACTUATOR_IMAGE: "ghcr.io/mchmarny/cluster/gke:v0.5.17@sha256:8bfffd412d98276ee1000a6c083514e6b28928416cc37602ec637ed3a2167c99" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Authenticate to GCP | |
| id: auth | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 | |
| with: | |
| workload_identity_provider: "projects/116689922666/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider" | |
| service_account: "github-actions@eidosx.iam.gserviceaccount.com" | |
| - name: Setup gcloud | |
| uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 | |
| - name: Reconcile GCP orphans | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GOOGLE_GHA_CREDS_PATH: ${{ steps.auth.outputs.credentials_file_path }} | |
| run: ./.github/scripts/uat-janitor.sh gcp | |
| aws: | |
| if: >- | |
| github.repository == 'nvidia/aicr' && | |
| github.ref == 'refs/heads/main' && | |
| (github.event_name == 'schedule' || inputs.cloud == 'all' || inputs.cloud == 'aws') | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| permissions: | |
| contents: read | |
| id-token: write | |
| actions: read | |
| env: | |
| AWS_REGION: "us-east-1" | |
| JANITOR_CONFIG: "tests/uat/aws/cluster-config.yaml" | |
| JANITOR_ACTUATOR_IMAGE: "ghcr.io/mchmarny/cluster/eks@sha256:8bc33d14e6e5659d242aa2cdbcd69fca389cfcdb9e94f7a7a1b82fd33176befa" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Authenticate to AWS | |
| uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 | |
| with: | |
| role-to-assume: "arn:aws:iam::615299774277:role/github-actions-role-aicr" | |
| aws-region: ${{ env.AWS_REGION }} | |
| role-session-name: GitHubActions-UAT-Janitor | |
| - name: Reconcile AWS orphans | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: ./.github/scripts/uat-janitor.sh aws | |
| # GB200 lives in a SEPARATE AWS account — the cross-account Capacity Block in | |
| # tenancy 897722687756 (tests/uat/aws/cluster-config-gb200.yaml), not the | |
| # standard 615299774277 the `aws` job above covers — so it needs its own role | |
| # assumption + config, or aws-gb200 orphans are never discovered or reaped. | |
| # PREREQUISITE: infra/uat-aws-account applied in 897722687756 (GitHub OIDC | |
| # provider + github-actions-role-aicr whose trust permits this workflow's ref). | |
| # Until then the auth step fails harmlessly and the first dry-run surfaces the gap. | |
| aws-gb200: | |
| if: >- | |
| github.repository == 'nvidia/aicr' && | |
| github.ref == 'refs/heads/main' && | |
| (github.event_name == 'schedule' || inputs.cloud == 'all' || inputs.cloud == 'aws') | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| permissions: | |
| contents: read | |
| id-token: write | |
| actions: read | |
| env: | |
| AWS_REGION: "us-east-1" | |
| JANITOR_CONFIG: "tests/uat/aws/cluster-config-gb200.yaml" | |
| JANITOR_ACTUATOR_IMAGE: "ghcr.io/mchmarny/cluster/eks@sha256:8bc33d14e6e5659d242aa2cdbcd69fca389cfcdb9e94f7a7a1b82fd33176befa" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Authenticate to AWS (GB200 account) | |
| uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 | |
| with: | |
| role-to-assume: "arn:aws:iam::897722687756:role/github-actions-role-aicr" | |
| aws-region: ${{ env.AWS_REGION }} | |
| role-session-name: GitHubActions-UAT-Janitor | |
| - name: Reconcile GB200 orphans | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: ./.github/scripts/uat-janitor.sh aws | |
| azure: | |
| if: >- | |
| github.repository == 'nvidia/aicr' && | |
| github.ref == 'refs/heads/main' && | |
| (github.event_name == 'schedule' || inputs.cloud == 'all' || inputs.cloud == 'azure') | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| permissions: | |
| contents: read | |
| id-token: write | |
| actions: read | |
| env: | |
| AZURE_SUBSCRIPTION_ID: "e88faa01-b4fd-49d3-b934-0ad9f9fca307" | |
| JANITOR_CONFIG: "tests/uat/azure/cluster-config.yaml" | |
| JANITOR_ACTUATOR_IMAGE: "ghcr.io/mchmarny/cluster/aks@sha256:70f049e45cd0025e0d2172503f9d0569baeabff27080919e2fa04f3339b9f9ee" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Authenticate to Azure | |
| uses: azure/login@f5d393ae46f8fde4be8b75f32e3fc50e654ad0ca # v3.0.1 | |
| with: | |
| client-id: "047f0427-5dcf-4e7c-97c3-9c71d92c5abb" | |
| tenant-id: "43083d15-7273-40c1-b7db-39efd9ccc17a" | |
| subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }} | |
| - name: Reconcile Azure orphans | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: ./.github/scripts/uat-janitor.sh azure |