ci(sdk): gate client API compatibility (NVIDIA#2060)

Signed-off-by: Tjark Gunnar Rasche <trasche@nvidia.com>

Author: tjrasche

.github/actions/README.md

@@ -13,6 +13,20 @@ executable bits or `./script.sh` invocation.
 
 ### Core CI/CD Actions
 
+#### `go-test/`
+
+**Purpose**: Set up Go and Helm, verify vendored dependencies, and run unit tests with race detection and coverage
+**When to use**: Go CI workflows that use the repository's `make test` target
+**Inputs**:
+- `go_version` (required): Go version to install
+- `coverage_report` (optional): Whether to generate a coverage report (default: "false")
+- `coverage_threshold` (optional): Minimum coverage percentage (default: empty)
+- `helm_version` (required): Helm version from `load-versions`
+- `apidiff_version` (optional): apidiff version from `load-versions`; when set, installs apidiff and runs `make api-diff` (default: empty, which skips both steps)
+
+Callers that set `apidiff_version` must check out full history with
+`fetch-depth: 0` so `make api-diff` can resolve a reachable stable release tag.
+
 #### `security-scan/`
 **Purpose**: Anchore/Grype vulnerability scanning with SARIF upload
 **When to use**: Security validation in CI/CD pipelines
@@ -339,12 +353,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          fetch-depth: 0
       - uses: ./.github/actions/load-versions
         id: versions
       - uses: ./.github/actions/go-test
         with:
           go_version: ${{ steps.versions.outputs.go }}
           helm_version: ${{ steps.versions.outputs.helm }}
+          apidiff_version: ${{ steps.versions.outputs.apidiff }}
           coverage_report: 'true'
       - uses: ./.github/actions/go-lint
         with:
@@ -360,12 +377,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          fetch-depth: 0
       - uses: ./.github/actions/load-versions
         id: versions
       - uses: ./.github/actions/go-test
         with:
           go_version: ${{ steps.versions.outputs.go }}
           helm_version: ${{ steps.versions.outputs.helm }}
+          apidiff_version: ${{ steps.versions.outputs.apidiff }}
       - uses: ./.github/actions/go-build-release
         id: release
         with:
@@ -437,3 +457,7 @@ To use these actions in other repositories:
     helm_version: 'v4.2.2'
     coverage_report: 'true'
 ```
+
+The cross-repository example intentionally omits `apidiff_version`. Repositories
+without AICR's `make api-diff` target retain the original test behavior because
+an empty `apidiff_version` skips the API compatibility steps.

.github/actions/go-test/action.yml

@@ -30,6 +30,10 @@ inputs:
   helm_version:
     description: 'Helm version pinned in .settings.yaml (testing_tools.helm, via load-versions) — required by the argocd-helm live-render tests, which fail (not skip) in CI when helm is missing'
     required: true
+  apidiff_version:
+    description: 'Optional apidiff version pinned in .settings.yaml (linting.apidiff, via load-versions); leave empty to skip the SDK API compatibility check'
+    required: false
+    default: ''
 
 runs:
   using: 'composite'
@@ -71,9 +75,23 @@ runs:
       shell: bash
       run: make test
 
+    - name: Install API-diff tool
+      if: inputs.apidiff_version != ''
+      shell: bash
+      env:
+        APIDIFF_VERSION: ${{ inputs.apidiff_version }}
+      run: |
+        set -euo pipefail
+        GOFLAGS= go install "golang.org/x/exp/cmd/apidiff@${APIDIFF_VERSION}"
+
     - name: Coverage Report
       if: inputs.coverage_report == 'true'
       uses: ./.github/actions/go-coverage
       with:
         coverage_file: ./coverage.out
         threshold: ${{ inputs.coverage_threshold }}
+
+    - name: Check SDK API compatibility
+      if: inputs.apidiff_version != ''
+      shell: bash
+      run: make api-diff

.github/actions/install-e2e-tools/action.yml

@@ -49,8 +49,16 @@ runs:
         chmod +x tools/setup-tools
         AUTO_MODE=true ./tools/setup-tools --skip-go --skip-docker
 
-    - name: Verify tool installations
+    - name: Report tool installations
       shell: bash
       run: |
         echo "=== Installed Tools ==="
-        make tools-check
+        # This action reports the shared development-tool state, but E2E owns
+        # its required-tool contract. Do not make future strict_exact tools
+        # outside that contract a prerequisite for every E2E workflow.
+        if make tools-check; then
+          tools_check_rc=0
+        else
+          tools_check_rc=$?
+          printf '%s\n' "::warning title=Non-gating tool check failed::make tools-check exited with status ${tools_check_rc}; E2E continues because its required-tool contract is narrower."
+        fi

.github/actions/load-versions/action.yml

@@ -31,6 +31,9 @@ outputs:
   golangci_lint:
     description: 'golangci-lint version'
     value: ${{ steps.versions.outputs.golangci_lint }}
+  apidiff:
+    description: 'API-diff version pinned in .settings.yaml'
+    value: ${{ steps.versions.outputs.apidiff }}
   addlicense:
     description: 'addlicense version'
     value: ${{ steps.versions.outputs.addlicense }}
@@ -171,6 +174,7 @@ runs:
 
         # Linting
         echo "golangci_lint=$(yq eval '.linting.golangci_lint' .settings.yaml)" >> $GITHUB_OUTPUT
+        echo "apidiff=$(yq eval '.linting.apidiff' .settings.yaml)" >> $GITHUB_OUTPUT
         echo "yamllint=$(yq eval '.linting.yamllint' .settings.yaml)" >> $GITHUB_OUTPUT
         echo "addlicense=$(yq eval '.linting.addlicense' .settings.yaml)" >> $GITHUB_OUTPUT
         echo "go_licenses=$(yq eval '.linting.go_licenses' .settings.yaml)" >> $GITHUB_OUTPUT
@@ -230,6 +234,7 @@ runs:
         echo "  ko: ${{ steps.versions.outputs.ko }}"
         echo "  crane: ${{ steps.versions.outputs.crane }}"
         echo "  golangci_lint: ${{ steps.versions.outputs.golangci_lint }}"
+        echo "  apidiff: ${{ steps.versions.outputs.apidiff }}"
         echo "  yamllint: ${{ steps.versions.outputs.yamllint }}"
         echo "  addlicense: ${{ steps.versions.outputs.addlicense }}"
         echo "  go_licenses: ${{ steps.versions.outputs.go_licenses }}"

.github/workflows/qualification.yaml

@@ -51,6 +51,7 @@ jobs:
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           ref: ${{ inputs.ref || '' }}
+          fetch-depth: 0
           persist-credentials: false
 
       - name: Load versions
@@ -64,6 +65,7 @@ jobs:
           coverage_report: ${{ inputs.coverage_report }}
           coverage_threshold: ${{ steps.versions.outputs.coverage_threshold }}
           helm_version: ${{ steps.versions.outputs.helm }}
+          apidiff_version: ${{ steps.versions.outputs.apidiff }}
 
   lint:
     name: Lint

.settings.yaml

@@ -29,6 +29,8 @@ build_tools:
 
 # Linting
 linting:
+  # renovate: datasource=go depName=golang.org/x/exp depType=linting
+  apidiff: 'v0.0.0-20260727155853-b88d891fe743'
   # renovate: datasource=github-releases depName=golangci/golangci-lint depType=linting
   golangci_lint: 'v2.12.2'
   # renovate: datasource=pypi depName=yamllint depType=linting

Makefile

@@ -288,8 +288,12 @@ scan: ## Scans for vulnerabilities with grype
 	echo "Running vulnerability scan..."; \
 	grype dir:. --config .grype.yaml --fail-on high --quiet
 
+.PHONY: api-diff
+api-diff: ## Checks pkg/client/v1 compatibility against the latest stable release
+	@bash tools/api-diff
+
 .PHONY: qualify
-qualify: test-coverage lint tuning-check e2e scan license-check ## Qualifies the codebase (test-coverage, lint, tuning-check, e2e, scan)
+qualify: test-coverage lint tuning-check e2e scan license-check api-diff ## Qualifies the codebase (test-coverage, lint, tuning-check, e2e, scan, API compatibility)
 	@echo "Codebase qualification completed"
 
 .PHONY: bom

docs/contributor/maintaining.md

@@ -27,6 +27,48 @@ The short form:
 
 Bi-weekly cadence; hotfix between cycles when a fix is critical.
 
+### SDK API Compatibility
+
+`make api-diff` compares the exported `pkg/client/v1` surface to the latest
+stable release tag. It runs through `make qualify` and the qualification
+workflow; additive changes are reported, while removals and incompatible type
+changes fail the gate.
+
+Local runs require the repository-pinned `apidiff` and `yq`; install them with
+`make tools-setup`. They also require full tag history and a stable release tag
+reachable from `HEAD`. The gate checks out that baseline in a temporary
+detached worktree, so it leaves the current working tree unchanged but adds
+checkout and filesystem I/O cost to `make qualify`.
+
+The gate compares declarations exported by `pkg/client/v1`. For external named
+types reached through transparent aliases, `apidiff` matches the target's
+package path and type name but does not recursively compare its definition. A
+change to such a target can therefore alter the facade contract without failing
+the gate. Reviewers must manually assess changes to aliased target types as
+changes to the stable facade until
+[#2019](https://github.com/NVIDIA/aicr/issues/2019) resolves this gap.
+
+To acknowledge an intentional break, first run `make api-diff`. Add a
+baseline-scoped entry to `pkg/client/v1/api-diff-exceptions.yaml` containing the
+reported baseline plus non-empty `issue`, `summary`, and `rationale` fields.
+Exactly one acknowledgement entry is allowed per baseline. Copy every reported
+incompatible line that begins with `- ` into `incompatible_changes`, omitting
+the `- ` prefix. The `incompatible_changes` list must exactly and completely
+match the command output; omissions and extras both fail the gate.
+
+The acknowledgement authorizes a break only for the active baseline, so keep it
+in place through the release that ships that break — it is what keeps the gate
+green until the release tag lands. Once the tag advances the baseline, the entry
+is obsolete and should be pruned. The gate is deliberately asymmetric about
+this. When the diff against the new baseline is clean there is nothing for a
+stale entry to authorize, so the gate only warns that the entry is prunable and
+still exits successfully; this is what lets the release pipeline and open pull
+requests stay green while the cleanup lands. When the diff is *not* clean, a
+stale entry is a hard failure: an acknowledgement scoped to an older baseline
+must never be accepted for a break against the current one. Prune the entry in a
+follow-up change after the release; Git history retains the release-notes record
+of the breaking change.
+
 ### Common Release Breakages
 
 **`goreleaser` fails with auth conflict.** `goreleaser` panics if both

docs/contributor/tests.md

@@ -483,6 +483,8 @@ half of the pipeline and skips deploy-side assertions.
 - `e2e` — the end-to-end pipeline runner.
 - `scan` — Grype vulnerability scan.
 - `license-check` — license header / dependency-license sweep.
+- `api-diff` — exported `pkg/client/v1` compatibility against the latest stable
+  release.
 
 CI runs the equivalent. If `make qualify` passes locally on the
 current branch, push CI will pass.

docs/integrator/public-api.md

@@ -56,6 +56,13 @@ The `github.com/NVIDIA/aicr/pkg/client/v1` package is Public (stable). Types
 reachable from this surface are either facade-owned structs or transparent
 aliases — the table below documents which.
 
+Transparent aliases extend that stable contract to their target types. The
+automated API-diff gate matches an external named target by package path and
+type name, but does not recursively compare the target's definition. Until
+[#2019](https://github.com/NVIDIA/aicr/issues/2019) resolves this limitation,
+changes to those definitions require manual compatibility review because they
+can affect facade consumers without failing the gate.
+
 | Facade symbol | Translates to/from | Notes |
 |---|---|---|
 | `aicr.Snapshot` | `pkg/snapshotter.Snapshot` | **Facade-owned struct**. Public fields are identifying metadata; full measurement payload is preserved in an unexported field for round-trip through `ValidateState`. Use `aicr.WrapSnapshot` to lift a `*snapshotter.Snapshot` loaded externally. |