feat(validation): detect missing GKE GPU NIC networks at deployment p… #47
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: KWOK Cluster Validation | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'recipes/**' | |
| - 'kwok/**' | |
| - 'tests/chainsaw/kwok/**' | |
| # The aicr binary that renders/bundles recipes is built from Go source; | |
| # a recipe-rendering regression or dependency bump must trigger KWOK e2e. | |
| - 'pkg/recipe/**' | |
| - 'pkg/bundler/**' | |
| - 'pkg/cli/**' | |
| - 'cmd/aicr/**' | |
| - 'go.mod' | |
| - 'go.sum' | |
| - 'vendor/**' | |
| - '.github/workflows/kwok-recipes.yaml' | |
| - '.github/workflows/kwok-test-run.yaml' | |
| - '.github/actions/kwok-test/**' | |
| - '!**.md' | |
| pull_request: | |
| branches: | |
| - main | |
| paths: | |
| - 'recipes/**' | |
| - 'kwok/**' | |
| - 'tests/chainsaw/kwok/**' | |
| # The aicr binary that renders/bundles recipes is built from Go source; | |
| # a recipe-rendering regression or dependency bump must trigger KWOK e2e. | |
| - 'pkg/recipe/**' | |
| - 'pkg/bundler/**' | |
| - 'pkg/cli/**' | |
| - 'cmd/aicr/**' | |
| - 'go.mod' | |
| - 'go.sum' | |
| - 'vendor/**' | |
| - '.github/workflows/kwok-recipes.yaml' | |
| - '.github/workflows/kwok-test-run.yaml' | |
| - '.github/actions/kwok-test/**' | |
| - '!**.md' | |
| schedule: | |
| # Nightly at 03:00 UTC — Tier 3 full matrix backstop | |
| - cron: '0 3 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| recipe: | |
| description: 'Specific recipe to test (leave empty to test all with service criteria)' | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| # PR concurrency: cancel in-progress runs for the same PR | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| discover: | |
| name: Discover Recipes | |
| if: github.event_name != 'schedule' || github.repository == 'nvidia/aicr' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| tier1_pairs: ${{ steps.classify.outputs.tier1_pairs }} | |
| tier2_pairs: ${{ steps.classify.outputs.tier2_pairs }} | |
| tier3_batches: ${{ steps.classify.outputs.tier3_batches }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| # Full checkout needed for diff-aware Tier 2 discovery | |
| fetch-depth: 0 | |
| - name: Script unit tests (kwok/scripts/lib) | |
| run: | | |
| # yq is required by profile-select_test.sh; fail closed here so a | |
| # runner image without yq cannot silently skip the assertions and | |
| # let the CI light lie about coverage (#1997). | |
| command -v yq >/dev/null || { echo "::error::yq is required for kwok/scripts/lib tests"; exit 1; } | |
| bash kwok/scripts/lib/sync-budget_test.sh | |
| bash kwok/scripts/lib/profile-select_test.sh | |
| bash kwok/scripts/run-all-recipes_test.sh | |
| - name: Classify recipes into tiers | |
| id: classify | |
| shell: bash | |
| env: | |
| DISPATCH_RECIPE: ${{ github.event.inputs.recipe }} | |
| run: | | |
| set -euo pipefail | |
| # Shared profile-selection helpers. Discovery filters out any | |
| # recipe whose (service, accelerator) has no matching KWOK profile | |
| # on disk, so the matrix never dispatches a job that would silently | |
| # "SKIP" and return green without validating anything (#1997). | |
| # The library defines read_criteria + profile_status (used below) | |
| # alongside the direct-path selector; keeping them there lets | |
| # profile-select_test.sh cover both wrappers. | |
| # shellcheck source=kwok/scripts/lib/profile-select.sh | |
| source kwok/scripts/lib/profile-select.sh | |
| PROFILES_ROOT="${GITHUB_WORKSPACE}/kwok/profiles" | |
| dropped="[]" # recipes filtered out of the matrix (no profile) | |
| # --- Deployer list: single source of truth (#1172) --- | |
| # Defined once, here (before dispatch, since dispatch also needs it). | |
| # Tier 1 and Tier 3 both consume it to build their {recipe, deployer} | |
| # pairs. Tier 2 stays helm-only by deliberate policy (see ADR-003 | |
| # "Tier 2 deployer coverage"). Keep .github/actions/kwok-test/action.yml's | |
| # deployer input doc-string in sync with this list (prose only, not enforced). | |
| deployers='["helm","argocd-oci","argocd-helm-oci","argocd-git","flux-oci","flux-git"]' | |
| # --- workflow_dispatch: test exactly the requested recipe --- | |
| if [[ -n "${DISPATCH_RECIPE}" ]]; then | |
| dispatch_overlay="recipes/overlays/${DISPATCH_RECIPE}.yaml" | |
| if [[ ! -f "${dispatch_overlay}" ]]; then | |
| echo "::error::Dispatch recipe '${DISPATCH_RECIPE}' has no overlay at ${dispatch_overlay}" | |
| exit 1 | |
| fi | |
| # workflow_dispatch is an explicit ask; an unmapped recipe here | |
| # must fail (not silently produce an empty matrix) so the | |
| # operator knows their request cannot be validated. | |
| # Capture rc separately: profile_status's return 1 on a fatal | |
| # path (malformed profile / ambiguous match) only manifests | |
| # as the substitution's rc, so an unchecked call would be | |
| # misread as the unmapped-skip case below. | |
| status=$(profile_status "${dispatch_overlay}" "${PROFILES_ROOT}") || exit 1 | |
| if [[ "${status}" == "${PROFILE_SELECT_RC_NO_MATCH}" ]]; then | |
| echo "::error::Dispatch recipe '${DISPATCH_RECIPE}' has no KWOK profile for its" | |
| echo "::error::(service, accelerator). Add one under kwok/profiles/<service>/ or" | |
| echo "::error::dispatch a different recipe." | |
| exit 1 | |
| fi | |
| single=$(jq -cn --arg r "${DISPATCH_RECIPE}" --argjson deployers "$deployers" \ | |
| '[ $deployers[] as $d | {recipe: $r, deployer: $d} ]') | |
| echo "tier1_pairs=${single}" >> "$GITHUB_OUTPUT" | |
| echo "tier2_pairs=[]" >> "$GITHUB_OUTPUT" | |
| echo "tier3_batches=[]" >> "$GITHUB_OUTPUT" | |
| echo "Manual dispatch: ${DISPATCH_RECIPE}" | |
| exit 0 | |
| fi | |
| # --- Scan all overlays and classify --- | |
| tier1="[]" # generic overlays (no accelerator specialization) | |
| all="[]" # every testable overlay | |
| for overlay in recipes/overlays/*.yaml; do | |
| name=$(basename "$overlay" .yaml) | |
| # Read BOTH criteria up front so type validation runs on the | |
| # accelerator even when the overlay would be skipped for | |
| # service reasons below — an ocp / empty-service recipe with | |
| # `accelerator: false` must still fail closed, since the | |
| # invalid type is a tree fault regardless of testability. | |
| # Same fail-closed contract as the direct path. | |
| service=$(read_criteria "$overlay" service) || exit 1 | |
| accel=$(read_criteria "$overlay" accelerator) || exit 1 | |
| # Skip non-testable overlays (no service, or OCP — needs OpenShift operators). | |
| if [[ -z "$service" || "$service" == "ocp" ]]; then | |
| continue | |
| fi | |
| # Drop recipes with no matching KWOK profile so the matrix cell | |
| # can't report green without running anything. Fatal selector | |
| # errors (ambiguous match, malformed profile) surface as | |
| # profile_status returning 1 — that only manifests as the | |
| # substitution's rc, so the outer script has to check it | |
| # explicitly and propagate, otherwise a broken tree gets | |
| # silently reclassified as an unmapped-skip and hidden. | |
| status=$(profile_status "${overlay}" "${PROFILES_ROOT}") || exit 1 | |
| if [[ "${status}" == "${PROFILE_SELECT_RC_NO_MATCH}" ]]; then | |
| dropped=$(echo "$dropped" | jq -c --arg r "$name" '. + [$r]') | |
| echo "::notice file=${overlay}::${name} filtered from KWOK matrix — no profile for its (service, accelerator)" | |
| continue | |
| fi | |
| all=$(echo "$all" | jq -c --arg r "$name" '. + [$r]') | |
| if [[ -z "$accel" ]]; then | |
| tier1=$(echo "$tier1" | jq -c --arg r "$name" '. + [$r]') | |
| fi | |
| done | |
| # --- Tier 2: diff-aware accelerator tests (PR only) --- | |
| tier2="[]" | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| base_ref="${{ github.event.pull_request.base.sha }}" | |
| head_ref="${{ github.event.pull_request.head.sha }}" | |
| if ! changed_files=$(git diff --name-only "${base_ref}..${head_ref}" 2>&1); then | |
| echo "::error::git diff failed — cannot compute Tier 2 affected overlays" | |
| echo "git diff output: ${changed_files}" | |
| exit 1 | |
| fi | |
| # If registry.yaml or base.yaml changed, all accelerator overlays are | |
| # potentially affected — promote the full accelerator set to Tier 2. | |
| promote_all=false | |
| if echo "$changed_files" | grep -qE '^recipes/(registry\.yaml|overlays/base\.yaml)$'; then | |
| promote_all=true | |
| echo "registry.yaml or base.yaml changed — all accelerator overlays promoted to Tier 2" | |
| fi | |
| # Build set of affected accelerator-specific overlays | |
| for overlay in recipes/overlays/*.yaml; do | |
| name=$(basename "$overlay" .yaml) | |
| service=$(read_criteria "$overlay" service) || exit 1 | |
| accel=$(read_criteria "$overlay" accelerator) || exit 1 | |
| # Only accelerator-specific, KWOK-testable overlays belong in Tier 2 | |
| if [[ -z "$service" || "$service" == "ocp" ]]; then | |
| continue | |
| fi | |
| if [[ -z "$accel" ]]; then | |
| continue | |
| fi | |
| # Same profile-availability filter as the main loop — an | |
| # unmapped accelerator recipe must not enter the matrix, or | |
| # its Tier 2 cell reports green after skipping (#1997). | |
| # Same substitution-rc trap as above: capture rc explicitly | |
| # so fatal selector errors propagate instead of being | |
| # silently reclassified as unmapped-skips. | |
| status=$(profile_status "${overlay}" "${PROFILES_ROOT}") || exit 1 | |
| if [[ "${status}" == "${PROFILE_SELECT_RC_NO_MATCH}" ]]; then | |
| continue | |
| fi | |
| # If registry.yaml or base.yaml changed, include all accelerator overlays | |
| if [[ "$promote_all" == "true" ]]; then | |
| tier2=$(echo "$tier2" | jq -c --arg r "$name" '. + [$r]') | |
| continue | |
| fi | |
| include=false | |
| # Rule 1: overlay file itself changed | |
| if echo "$changed_files" | grep -qF "recipes/overlays/${name}.yaml"; then | |
| include=true | |
| fi | |
| # Rule 2: check if any ancestor overlay in the base chain changed | |
| if [[ "$include" == "false" ]]; then | |
| current="$overlay" | |
| while true; do | |
| parent=$(yq eval '.spec.base // ""' "$current" 2>/dev/null || true) | |
| if [[ -z "$parent" || "$parent" == "null" ]]; then | |
| break | |
| fi | |
| if echo "$changed_files" | grep -qF "recipes/overlays/${parent}.yaml"; then | |
| include=true | |
| break | |
| fi | |
| current="recipes/overlays/${parent}.yaml" | |
| if [[ ! -f "$current" ]]; then | |
| break | |
| fi | |
| done | |
| fi | |
| # Rule 3: a component values file referenced by this overlay changed | |
| if [[ "$include" == "false" ]]; then | |
| # Collect valuesFile references from this overlay and its base chain | |
| values_files="" | |
| current="$overlay" | |
| while true; do | |
| vf=$(yq eval '.spec.componentRefs[].valuesFile // ""' "$current" 2>/dev/null | grep -v '^$' || true) | |
| if [[ -n "$vf" ]]; then | |
| values_files="${values_files}"$'\n'"${vf}" | |
| fi | |
| parent=$(yq eval '.spec.base // ""' "$current" 2>/dev/null || true) | |
| if [[ -z "$parent" || "$parent" == "null" ]]; then | |
| break | |
| fi | |
| current="recipes/overlays/${parent}.yaml" | |
| if [[ ! -f "$current" ]]; then | |
| break | |
| fi | |
| done | |
| # Check if any referenced values file changed | |
| for vf in $values_files; do | |
| # valuesFile paths are relative to recipes/ | |
| if echo "$changed_files" | grep -qF "recipes/${vf}"; then | |
| include=true | |
| break | |
| fi | |
| done | |
| fi | |
| if [[ "$include" == "true" ]]; then | |
| tier2=$(echo "$tier2" | jq -c --arg r "$name" '. + [$r]') | |
| fi | |
| done | |
| fi | |
| # --- Tier 3: full matrix (all testable overlays) --- | |
| tier3="$all" | |
| tier1_pairs=$(jq -cn --argjson recipes "$tier1" --argjson deployers "$deployers" ' | |
| [ $recipes[] as $r | $deployers[] as $d | {recipe: $r, deployer: $d} ] | |
| ') | |
| tier2_pairs=$(jq -cn --argjson recipes "$tier2" ' | |
| [ $recipes[] as $r | {recipe: $r, deployer: "helm"} ] | |
| ') | |
| # --- Tier 3 batching --- | |
| # GitHub caps a single job's matrix at 256 configurations. Tier 3 | |
| # crosses every testable recipe with every deployer, so the raw | |
| # cross-product (recipes × deployers) outgrew the cap. Split the | |
| # {recipe, deployer} pairs into batches of <= TIER3_BATCH_SIZE; the | |
| # caller fans each batch out to the shared kwok-test-run.yaml | |
| # reusable workflow (one call per batch), keeping every shard's | |
| # matrix under the limit. | |
| readonly TIER3_BATCH_SIZE=200 # headroom under GitHub's 256 cap | |
| # Fail closed if the batch size is ever raised past the hard limit — | |
| # a too-large batch would otherwise resurface the opaque | |
| # "produced N configurations which exceeds the maximum of 256" error. | |
| if (( TIER3_BATCH_SIZE > 256 )); then | |
| echo "::error::TIER3_BATCH_SIZE=${TIER3_BATCH_SIZE} exceeds GitHub's 256-configuration matrix limit" | |
| exit 1 | |
| fi | |
| tier3_batches=$(jq -cn \ | |
| --argjson recipes "$tier3" \ | |
| --argjson deployers "$deployers" \ | |
| --argjson size "$TIER3_BATCH_SIZE" ' | |
| [ $recipes[] as $r | $deployers[] as $d | {recipe: $r, deployer: $d} ] | |
| | [ range(0; length; $size) as $i | |
| # id = zero-based batch index; used as the concurrency-group suffix | |
| | {id: ($i / $size | floor), pairs: .[$i:$i+$size]} ] | |
| ') | |
| # --- Output --- | |
| echo "tier1_pairs=${tier1_pairs}" >> "$GITHUB_OUTPUT" | |
| echo "tier2_pairs=${tier2_pairs}" >> "$GITHUB_OUTPUT" | |
| echo "tier3_batches=${tier3_batches}" >> "$GITHUB_OUTPUT" | |
| tier1_count=$(echo "$tier1_pairs" | jq 'length') | |
| if (( tier1_count > 200 )); then | |
| echo "::warning::Tier 1 contains ${tier1_count} pairs and is approaching GitHub's 256-configuration matrix limit" | |
| fi | |
| if (( tier1_count > 256 )); then | |
| echo "::error::Tier 1 has grown to ${tier1_count} pairs (>256) — exceeds GitHub's matrix cap. Batch Tier 1 like Tier 3, or tighten the generic-overlay criteria." | |
| exit 1 | |
| fi | |
| tier2_count=$(echo "$tier2_pairs" | jq 'length') | |
| tier3_pairs=$(echo "$tier3_batches" | jq '[.[].pairs[]] | length') | |
| tier3_batch_count=$(echo "$tier3_batches" | jq 'length') | |
| echo "Tier 1 (generic): $(echo "$tier1" | jq 'length') recipe(s) × $(echo "$deployers" | jq 'length') deployer(s) = ${tier1_count} pair(s)" | |
| echo "Tier 2 (diff-aware): $(echo "$tier2" | jq 'length') recipe(s) × 1 deployer (helm) = ${tier2_count} pair(s)" | |
| echo "Tier 3 (full matrix): $(echo "$tier3" | jq 'length') recipe(s) × $(echo "$deployers" | jq 'length') deployer(s) = ${tier3_pairs} pair(s) in ${tier3_batch_count} batch(es)" | |
| dropped_count=$(echo "$dropped" | jq 'length') | |
| if (( dropped_count > 0 )); then | |
| echo "Filtered from matrix: ${dropped_count} recipe(s) with no matching KWOK profile — $(echo "$dropped" | jq -r 'join(", ")')" | |
| fi | |
| # Thin caller of the shared kwok-test-run.yaml reusable workflow (#1172). | |
| # tier1_pairs is well under 256 entries, so it's passed in a single call — | |
| # no batching needed. | |
| test-tier1: | |
| name: 'Tier 1' | |
| needs: discover | |
| if: >- | |
| github.event_name != 'schedule' && | |
| needs.discover.outputs.tier1_pairs != '[]' && | |
| needs.discover.outputs.tier1_pairs != '' | |
| uses: ./.github/workflows/kwok-test-run.yaml | |
| with: | |
| pairs: ${{ needs.discover.outputs.tier1_pairs }} | |
| # ── Tier 2: diff-aware accelerator tests (PR only, conditional) ── | |
| # Helm-only by deliberate policy — see ADR-003 "Tier 2 deployer coverage". | |
| test-tier2: | |
| name: 'Tier 2' | |
| needs: discover | |
| if: >- | |
| github.event_name == 'pull_request' && | |
| needs.discover.outputs.tier2_pairs != '[]' && | |
| needs.discover.outputs.tier2_pairs != '' | |
| uses: ./.github/workflows/kwok-test-run.yaml | |
| with: | |
| pairs: ${{ needs.discover.outputs.tier2_pairs }} | |
| # ── Tier 3: full matrix (push to main + nightly schedule) ── | |
| # The recipe × deployer cross-product exceeds GitHub's 256-config matrix cap, | |
| # so discover splits it into batches and we fan each batch out to the shared | |
| # kwok-test-run.yaml reusable workflow (one call per batch, each <= 256). | |
| # Per ADR-003: the concurrency group is keyed by SHA so successive merges to | |
| # main never cancel in-flight Tier 3 runs; the batch id keeps every shard of a | |
| # single run in its own group so they all run in parallel. | |
| test-tier3: | |
| name: 'Tier 3' | |
| needs: discover | |
| concurrency: | |
| group: kwok-tier3-${{ github.sha }}-${{ matrix.batch.id }} | |
| cancel-in-progress: false | |
| if: >- | |
| (github.event_name == 'push' || (github.event_name == 'schedule' && github.repository == 'nvidia/aicr')) && | |
| needs.discover.outputs.tier3_batches != '[]' && | |
| needs.discover.outputs.tier3_batches != '' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| batch: ${{ fromJSON(needs.discover.outputs.tier3_batches) }} | |
| uses: ./.github/workflows/kwok-test-run.yaml | |
| with: | |
| pairs: ${{ toJSON(matrix.batch.pairs) }} | |
| # ── Summary: aggregate all tiers ── | |
| summary: | |
| name: KWOK Test Summary | |
| needs: [test-tier1, test-tier2, test-tier3] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| if: always() | |
| steps: | |
| - name: Check test results | |
| run: | | |
| echo "## KWOK Cluster Validation Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| tier1="${{ needs.test-tier1.result }}" | |
| tier2="${{ needs.test-tier2.result }}" | |
| tier3="${{ needs.test-tier3.result }}" | |
| echo "| Tier | Result |" >> $GITHUB_STEP_SUMMARY | |
| echo "|------|--------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Tier 1 (generic) | ${tier1} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Tier 2 (diff-aware) | ${tier2} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Tier 3 (full matrix) | ${tier3} |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| failed=false | |
| # Tier 1 must pass (unless skipped on schedule-only with no recipes) | |
| if [[ "$tier1" == "failure" || "$tier1" == "cancelled" ]]; then | |
| echo "Tier 1 (generic) failed or was cancelled" >> $GITHUB_STEP_SUMMARY | |
| failed=true | |
| fi | |
| # Tier 2 must pass when it runs (skipped is OK — means no diff-affected overlays) | |
| if [[ "$tier2" == "failure" || "$tier2" == "cancelled" ]]; then | |
| echo "Tier 2 (diff-aware) failed or was cancelled" >> $GITHUB_STEP_SUMMARY | |
| failed=true | |
| fi | |
| # Tier 3 must pass on push/schedule (skipped is OK on PR) | |
| if [[ "$tier3" == "failure" || "$tier3" == "cancelled" ]]; then | |
| echo "Tier 3 (full matrix) failed or was cancelled" >> $GITHUB_STEP_SUMMARY | |
| failed=true | |
| fi | |
| if [[ "$failed" == "true" ]]; then | |
| exit 1 | |
| fi | |
| echo "All recipe validations passed" >> $GITHUB_STEP_SUMMARY |