fix(cli): fix kubeconfig wiring for validate jobs

Signed-off-by: Tjark Gunnar Rasche <trasche@nvidia.com>

Author: tjrasche

docs/user/cli-reference.md

@@ -821,7 +821,7 @@ aicr validate [flags]
 | `--fail-on-error` | | bool | true | Exit with non-zero status if any constraint fails |
 | `--fail-fast` | | bool | false | Stop after the first phase that fails. By default all phases run and produce results. |
 | `--output` | `-o` | string | stdout | Output destination: file path, ConfigMap URI (`cm://namespace/name`), or stdout |
-| `--kubeconfig` | `-k` | string | ~/.kube/config | Path to kubeconfig file (used when `--recipe`, `--snapshot`, or `--output` is a ConfigMap URI) |
+| `--kubeconfig` | `-k` | string | ~/.kube/config | Path to kubeconfig file for the entire validation run: live snapshot capture, ConfigMap recipe/snapshot reads, validation namespace/RBAC/ConfigMaps/Jobs/results/cleanup, and ConfigMap output. Omit it to use standard Kubernetes client discovery. |
 | `--namespace` | `-n` | string | aicr-validation | Kubernetes namespace for validation Job deployment |
 | `--image` | | string | ghcr.io/nvidia/aicr:latest | Container image for validation Job |
 | `--image-pull-secret` | | string[] | | Image pull secrets for private registries (repeatable) |
@@ -937,7 +937,7 @@ aicr validate \
   --snapshot snapshot.yaml \
   --phase performance
 
-# With custom kubeconfig
+# Run every cluster operation against a custom target cluster
 aicr validate \
   --recipe recipe.yaml \
   --snapshot cm://gpu-operator/aicr-snapshot \

pkg/cli/validate.go

@@ -24,16 +24,13 @@ import (
 
 	"github.com/urfave/cli/v3"
 	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	aicr "github.com/NVIDIA/aicr/pkg/client/v1"
 	"github.com/NVIDIA/aicr/pkg/config"
 	"github.com/NVIDIA/aicr/pkg/defaults"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 
 	"github.com/NVIDIA/aicr/pkg/errors"
 	"github.com/NVIDIA/aicr/pkg/evidence/cncf"
-	k8sclient "github.com/NVIDIA/aicr/pkg/k8s/client"
 	"github.com/NVIDIA/aicr/pkg/serializer"
 	"github.com/NVIDIA/aicr/pkg/snapshotter"
 	"github.com/NVIDIA/aicr/pkg/validator"
@@ -152,21 +149,10 @@ func resolveValidateTolerations(cmd *cli.Command, resolved *config.ValidateResol
 	return resolved.Tolerations, nil
 }
 
-// deployAgentForValidation deploys an agent to capture a snapshot and returns the Snapshot.
-// Creates the namespace if it does not exist.
-func deployAgentForValidation(ctx context.Context, cfg *validateAgentConfig) (*snapshotter.Snapshot, string, error) {
-	// Ensure namespace exists before deploying the agent Job.
-	clientset, _, err := k8sclient.GetKubeClient()
-	if err != nil {
-		return nil, "", errors.PropagateOrWrap(err, errors.ErrCodeInternal, "failed to create kubernetes client")
-	}
-	ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: cfg.namespace}}
-	if _, nsErr := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}); nsErr != nil {
-		if !apierrors.IsAlreadyExists(nsErr) {
-			return nil, "", errors.Wrap(errors.ErrCodeInternal, "failed to create namespace", nsErr)
-		}
-	}
-
+// deployAgentForValidation deploys an agent to capture a snapshot and returns
+// the Snapshot. The snapshotter owns namespace creation using the same
+// run-scoped kubeconfig as the rest of the agent lifecycle.
+func deployAgentForValidation(ctx context.Context, cfg *validateAgentConfig) (*snapshotter.Snapshot, error) {
 	agentConfig := &snapshotter.AgentConfig{
 		Kubeconfig:         cfg.kubeconfig,
 		Namespace:          cfg.namespace,
@@ -185,20 +171,19 @@ func deployAgentForValidation(ctx context.Context, cfg *validateAgentConfig) (*s
 
 	snap, err := snapshotter.DeployAndGetSnapshot(ctx, agentConfig)
 	if err != nil {
-		return nil, "", errors.Wrap(errors.ErrCodeInternal, "failed to capture snapshot", err)
+		return nil, errors.PropagateOrWrap(err, errors.ErrCodeInternal, "failed to capture snapshot")
 	}
 
-	source := fmt.Sprintf("agent:%s/%s", cfg.namespace, cfg.jobName)
-	return snap, source, nil
+	return snap, nil
 }
 
 // validationConfig holds all parameters for a validation run.
 type validationConfig struct {
 	// Input
 	phases []validator.Phase
 
-	// Kubeconfig path; propagated to ConfigMap reads/writes so a single
-	// validate invocation can target a non-default cluster end-to-end.
+	// Kubeconfig path; propagated to every Kubernetes operation so a single
+	// validate invocation targets one cluster end-to-end.
 	kubeconfig string
 
 	// Output
@@ -261,6 +246,7 @@ func runValidation(
 	// validator.With* calls); the image/commit overrides are passed verbatim
 	// (empty string is the validator's "unset" sentinel).
 	opts := []aicr.ValidateOption{
+		aicr.WithValidationKubeconfig(cfg.kubeconfig),
 		aicr.WithValidationNamespace(cfg.validationNamespace),
 		aicr.WithValidationRunID(runID),
 		aicr.WithValidationCleanup(cfg.cleanup),
@@ -733,7 +719,7 @@ Run validation without failing on check errors (informational mode):
 				agentCfg := parseValidateAgentConfig(cmd, resolved, shared)
 
 				var deployErr error
-				snap, _, deployErr = deployAgentForValidation(ctx, agentCfg)
+				snap, deployErr = deployAgentForValidation(ctx, agentCfg)
 				if deployErr != nil {
 					return deployErr
 				}

pkg/cli/validate_test.go

@@ -16,6 +16,7 @@ package cli
 
 import (
 	"context"
+	stderrors "errors"
 	"os"
 	"path/filepath"
 	"slices"
@@ -24,9 +25,107 @@ import (
 
 	"github.com/urfave/cli/v3"
 
+	aicr "github.com/NVIDIA/aicr/pkg/client/v1"
+	"github.com/NVIDIA/aicr/pkg/errors"
+	"github.com/NVIDIA/aicr/pkg/serializer"
+	"github.com/NVIDIA/aicr/pkg/snapshotter"
 	v1 "github.com/NVIDIA/aicr/pkg/validator/v1"
 )
 
+// isolateValidationKubeconfigs gives validation two different invalid paths:
+// target is the explicit --kubeconfig value under test, while fallback is what
+// default discovery would choose. Both fail before any Kubernetes API call,
+// keeping tests hermetic while making a regression to the default client
+// observable in the resulting error.
+func isolateValidationKubeconfigs(t *testing.T) (target, fallback string) {
+	t.Helper()
+
+	dir := t.TempDir()
+	target = filepath.Join(dir, "target-kubeconfig")
+	fallback = filepath.Join(dir, "default-kubeconfig")
+	t.Setenv("KUBECONFIG", fallback)
+	t.Setenv("HOME", t.TempDir())
+	t.Setenv("USERPROFILE", t.TempDir())
+	t.Setenv("KUBERNETES_SERVICE_HOST", "")
+	t.Setenv("KUBERNETES_SERVICE_PORT", "")
+	return target, fallback
+}
+
+func assertExplicitKubeconfigError(t *testing.T, err error, target, fallback string) {
+	t.Helper()
+
+	if err == nil {
+		t.Fatal("error = nil, want explicit kubeconfig error")
+	}
+	if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) {
+		t.Errorf("error = %v, want ErrCodeInvalidRequest", err)
+	}
+	if !strings.Contains(err.Error(), target) {
+		t.Errorf("error = %v, want explicit kubeconfig path %q", err, target)
+	}
+	if strings.Contains(err.Error(), fallback) {
+		t.Errorf("error = %v, unexpectedly used default kubeconfig %q", err, fallback)
+	}
+}
+
+// TestRunValidationPropagatesKubeconfigToValidator pins the missing CLI facade
+// wiring: the kubeconfig used for inputs and output must also select the client
+// used for validation namespace, RBAC, ConfigMaps, Jobs, results, and cleanup.
+func TestRunValidationPropagatesKubeconfigToValidator(t *testing.T) {
+	target, fallback := isolateValidationKubeconfigs(t)
+
+	const recipeYAML = `kind: RecipeResult
+apiVersion: aicr.run/v1alpha2
+metadata:
+  version: test
+componentRefs:
+  - name: gpu-operator
+    type: helm
+    chart: gpu-operator
+    source: https://helm.ngc.nvidia.com/nvidia
+    version: v25.3.4
+deploymentOrder:
+  - gpu-operator
+`
+	recipePath := filepath.Join(t.TempDir(), "recipe.yaml")
+	if err := os.WriteFile(recipePath, []byte(recipeYAML), 0o600); err != nil {
+		t.Fatalf("write recipe: %v", err)
+	}
+
+	client, err := aicr.NewClient(aicr.WithRecipeSource(aicr.EmbeddedSource()))
+	if err != nil {
+		t.Fatalf("NewClient: %v", err)
+	}
+	t.Cleanup(func() { _ = client.Close() })
+
+	rec, err := client.LoadRecipe(t.Context(), recipePath, "")
+	if err != nil {
+		t.Fatalf("LoadRecipe: %v", err)
+	}
+
+	err = runValidation(t.Context(), client, rec, &snapshotter.Snapshot{}, validationConfig{
+		kubeconfig:          target,
+		output:              filepath.Join(t.TempDir(), "report.yaml"),
+		outFormat:           serializer.FormatYAML,
+		validationNamespace: "aicr-validation",
+		cleanup:             true,
+	})
+	assertExplicitKubeconfigError(t, err, target, fallback)
+}
+
+// TestDeployAgentForValidationUsesExplicitKubeconfigFromStart ensures live
+// snapshot capture does not pre-create its namespace through the default
+// client before handing the explicit path to snapshotter.
+func TestDeployAgentForValidationUsesExplicitKubeconfigFromStart(t *testing.T) {
+	target, fallback := isolateValidationKubeconfigs(t)
+
+	_, err := deployAgentForValidation(t.Context(), &validateAgentConfig{
+		kubeconfig: target,
+		namespace:  "aicr-validation",
+	})
+	assertExplicitKubeconfigError(t, err, target, fallback)
+}
+
 // TestResolveCNCFAllocationPolicy exercises the #1629 policy threading for
 // --cncf-submission runs: no recipe context resolves to an empty policy
 // (standalone runs keep the evidence script's capability detection), a