feat(sdk): add support for custom kubeconfig paths for validation jobs to SDK

Signed-off-by: Tjark Gunnar Rasche <trasche@nvidia.com>

Author: tjrasche

docs/integrator/go-library.md

@@ -112,9 +112,12 @@ if err != nil {
 }
 
 // ValidateState runs the validation phases against the resolved recipe +
-// observed snapshot. With no WithValidationPhases option it runs all three
+// observed snapshot. Pass the same kubeconfig you used for snapshot collection
+// so that namespace, RBAC, ConfigMap, validator Job, and result operations all
+// target that cluster. With no WithValidationPhases option it runs all three
 // phases (Deployment, Conformance, Performance) in canonical order.
-results, err := client.ValidateState(ctx, result, snap)
+results, err := client.ValidateState(ctx, result, snap,
+	aicr.WithValidationKubeconfig("/path/to/target-kubeconfig"))
 if err != nil {
 	log.Fatalf("validate state: %v", err)
 }
@@ -123,6 +126,13 @@ for _, r := range results {
 }
 ```
 
+When `WithValidationKubeconfig` is omitted or passed an empty string,
+`ValidateState` uses the shared default Kubernetes client and its standard
+discovery chain: `KUBECONFIG`, `~/.kube/config`, then in-cluster configuration.
+When an explicit path is provided, the SDK reloads that kubeconfig and creates a
+fresh client for each validation run. The run reuses that client for all of its
+Kubernetes operations.
+
 The `recipe` argument to `ValidateState` MUST be the `*RecipeResult`
 returned by the same Client's `ResolveRecipe` (or `LoadRecipe`) call —
 the unexported internal recipe state is required for constraint
@@ -169,8 +179,8 @@ reports as "skipped - no-cluster mode" and no Kubernetes resources
 are created. Other facade options
 (`WithValidationNamespace`, `WithValidationRunID`,
 `WithValidationCleanup`, `WithValidationImagePullSecrets`,
-`WithValidationTolerations`, `WithValidationNodeSelector`) cover the
-production-controller knobs.
+`WithValidationTolerations`, `WithValidationNodeSelector`,
+`WithValidationKubeconfig`) cover the production-controller knobs.
 
 ## Recipe sources
 

pkg/client/v1/aicr.go

@@ -1207,10 +1207,11 @@ func (c *Client) CollectSnapshot(ctx context.Context, cfg *AgentConfig) (*Snapsh
 // from unit tests so no Kubernetes resources are created and every
 // check reports as "skipped". WithValidationNamespace, WithValidationRunID,
 // WithValidationCleanup, WithValidationTolerations,
-// WithValidationNodeSelector, and WithValidationPhases cover the
-// production-controller knobs. The validator catalog loads through this
-// Client's own DataProvider, so a Client built from FilesystemSource
-// validates against that recipe source rather than the package global.
+// WithValidationNodeSelector, WithValidationKubeconfig, and
+// WithValidationPhases cover the production-controller knobs. The validator
+// catalog loads through this Client's own DataProvider, so a Client built from
+// FilesystemSource validates against that recipe source rather than the
+// package global.
 //
 // Errors:
 //   - ErrCodeInvalidRequest when the Client, recipe, or snap is nil,

pkg/client/v1/aicr_internal_test.go

@@ -656,6 +656,24 @@ func TestWithValidationTolerations_ExplicitNilOverrides(t *testing.T) {
 	}
 }
 
+// TestWithValidationKubeconfig_RoundTrip proves the facade-owned option is
+// translated to validator.WithKubeconfig without relying on process-global
+// KUBECONFIG state.
+func TestWithValidationKubeconfig_RoundTrip(t *testing.T) {
+	t.Parallel()
+
+	const path = "/path/to/target-kubeconfig"
+	cfg := buildValidateConfig([]ValidateOption{WithValidationKubeconfig(path)})
+	if cfg.kubeconfig != path {
+		t.Fatalf("kubeconfig = %q, want %q", cfg.kubeconfig, path)
+	}
+
+	v := validator.New(validateOptionsFromConfig(cfg)...)
+	if v.Kubeconfig != path {
+		t.Errorf("validator.Kubeconfig = %q, want %q", v.Kubeconfig, path)
+	}
+}
+
 // TestWithValidationTimeout_OptIn pins FIX D: WithValidationTimeout captures
 // a pointer-wrapped duration so the ValidateState switch can distinguish
 // unset (nil → default 60m), explicit 0 (no facade cap), and explicit >0.

pkg/client/v1/options.go

@@ -46,6 +46,7 @@ type ValidateOption func(*validateConfig)
 // nil to mean unset because the empty slice has different semantics
 // (e.g., "no tolerations at all" vs "no override; use default").
 type validateConfig struct {
+	kubeconfig       string
 	namespace        *string
 	runID            *string
 	cleanup          *bool
@@ -116,7 +117,10 @@ func buildValidateConfig(opts []ValidateOption) *validateConfig {
 // here and zero edits on the facade surface. phases is intentionally NOT
 // translated here — it is passed directly to ValidatePhases by the caller.
 func validateOptionsFromConfig(cfg *validateConfig) []validator.Option {
-	out := make([]validator.Option, 0, 11)
+	out := make([]validator.Option, 0, 12)
+	if cfg.kubeconfig != "" {
+		out = append(out, validator.WithKubeconfig(cfg.kubeconfig))
+	}
 	if cfg.namespace != nil {
 		out = append(out, validator.WithNamespace(*cfg.namespace))
 	}
@@ -157,6 +161,16 @@ func validateOptionsFromConfig(cfg *validateConfig) []validator.Option {
 	return out
 }
 
+// WithValidationKubeconfig sets an explicit, run-scoped kubeconfig path for
+// every Kubernetes API operation performed by Client.ValidateState, including
+// namespace, RBAC, ConfigMap, validator Job, and result operations. The file is
+// reloaded for each validation run. Empty uses the shared default Kubernetes
+// client and its standard KUBECONFIG, ~/.kube/config, then in-cluster discovery
+// chain.
+func WithValidationKubeconfig(kubeconfig string) ValidateOption {
+	return func(c *validateConfig) { c.kubeconfig = kubeconfig }
+}
+
 // WithValidationNamespace sets the Kubernetes namespace where
 // validation Jobs run. Default: "aicr-validation".
 func WithValidationNamespace(namespace string) ValidateOption {

pkg/client/v1/stability_test.go

@@ -114,6 +114,7 @@ func TestStability_Validate(t *testing.T) {
 	// Element type pins each WithValidation* return to aicr.ValidateOption;
 	// dropping or retyping any factory becomes a compile error.
 	_ = []aicr.ValidateOption{
+		aicr.WithValidationKubeconfig("/path/to/kubeconfig"),
 		aicr.WithValidationNamespace("ns"),
 		aicr.WithValidationRunID("rid"),
 		aicr.WithValidationCleanup(true),

pkg/validator/options.go

@@ -38,6 +38,16 @@ func WithCommit(commit string) Option {
 	}
 }
 
+// WithKubeconfig sets an explicit, run-scoped kubeconfig path for all
+// Kubernetes API operations performed by the validation run. The file is
+// reloaded for each run. Empty uses the shared default Kubernetes client and
+// its standard KUBECONFIG, ~/.kube/config, then in-cluster discovery chain.
+func WithKubeconfig(kubeconfig string) Option {
+	return func(v *Validator) {
+		v.Kubeconfig = kubeconfig
+	}
+}
+
 // WithNamespace sets the Kubernetes namespace for validation Jobs.
 // Default: "aicr-validation".
 func WithNamespace(namespace string) Option {

pkg/validator/types.go

@@ -36,6 +36,11 @@ type Validator struct {
 	// dev-build validator images to SHA-tagged images pushed by on-push CI.
 	Commit string
 
+	// Kubeconfig is an optional, run-scoped path overriding the Kubernetes
+	// client configuration used for validation namespace, RBAC, ConfigMap, Job,
+	// and result operations. Empty uses the shared default client.
+	Kubeconfig string
+
 	// Namespace is the Kubernetes namespace for validation Jobs.
 	Namespace string
 
@@ -80,6 +85,10 @@ type Validator struct {
 	// dataProvider supplies the recipe data files used to load the validator
 	// catalog. When nil, catalog.Load falls back to the package-global provider.
 	dataProvider recipe.DataProvider
+
+	// kubeClientFactory is an internal test seam for explicit-path client
+	// creation. Production validators leave it nil and use BuildKubeClient.
+	kubeClientFactory kubeClientFactory
 }
 
 // PhaseResult is the outcome of running all validators in a single phase.

pkg/validator/validator.go

@@ -96,6 +96,11 @@ type clusterState struct {
 	stopCh    chan struct{}
 }
 
+// kubeClientFactory constructs a run-scoped Kubernetes client for an explicit
+// kubeconfig path. Tests inject this seam to verify explicit-path propagation
+// without reading kubeconfig files or contacting a live cluster.
+type kubeClientFactory func(kubeconfig string) (kubernetes.Interface, error)
+
 // prepareCluster sets up namespace, RBAC, data ConfigMaps, and informer factory.
 // The caller must close stopCh and handle cleanup deferrals.
 func (v *Validator) prepareCluster(
@@ -107,7 +112,22 @@ func (v *Validator) prepareCluster(
 	// Use PropagateOrWrap so a coded inner error (e.g. an invalid kubeconfig
 	// classified as a deterministic config error) survives instead of being
 	// blanket-relabeled ErrCodeInternal, which would mask it as retryable.
-	clientset, _, err := k8sclient.GetKubeClient()
+	var clientset kubernetes.Interface
+	var err error
+	kubeconfig := strings.TrimSpace(v.Kubeconfig)
+	switch {
+	case kubeconfig == "":
+		// With no per-run override, use the package-wide default client so all
+		// consumers retain standard discovery and connection reuse semantics.
+		clientset, _, err = k8sclient.GetKubeClient()
+	case v.kubeClientFactory != nil:
+		clientset, err = v.kubeClientFactory(kubeconfig)
+	default:
+		// Explicit overrides are run-scoped: reload the file instead of retaining
+		// the client in the process-wide path cache. clusterState reuses this client
+		// for every phase and cleanup operation within the current run.
+		clientset, _, err = k8sclient.BuildKubeClient(kubeconfig)
+	}
 	if err != nil {
 		return nil, errors.PropagateOrWrap(err, errors.ErrCodeInternal, "failed to create kubernetes client")
 	}

pkg/validator/validator_test.go

@@ -16,20 +16,23 @@ package validator
 
 import (
 	"context"
+	stderrors "errors"
 	"fmt"
 	"io/fs"
 	"path/filepath"
 	"testing"
 
 	"gopkg.in/yaml.v3"
 
+	"github.com/NVIDIA/aicr/pkg/errors"
 	"github.com/NVIDIA/aicr/pkg/recipe"
 	"github.com/NVIDIA/aicr/pkg/snapshotter"
 	"github.com/NVIDIA/aicr/pkg/validator/catalog"
 	"github.com/NVIDIA/aicr/pkg/validator/ctrf"
 	v1 "github.com/NVIDIA/aicr/pkg/validator/v1"
 	"github.com/NVIDIA/aicr/recipes"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes"
 )
 
 func TestNewDefaults(t *testing.T) {
@@ -47,6 +50,9 @@ func TestNewDefaults(t *testing.T) {
 	if v.NoCluster {
 		t.Error("NoCluster should default to false")
 	}
+	if v.Kubeconfig != "" {
+		t.Errorf("Kubeconfig = %q, want empty", v.Kubeconfig)
+	}
 	if len(v.Tolerations) != 1 || v.Tolerations[0].Operator != corev1.TolerationOpExists {
 		t.Errorf("Tolerations should default to tolerate-all, got %v", v.Tolerations)
 	}
@@ -59,6 +65,7 @@ func TestNewWithOptions(t *testing.T) {
 	v := New(
 		WithVersion("1.0.0"),
 		WithCommit("abc1234"),
+		WithKubeconfig("/path/to/kubeconfig"),
 		WithNamespace("custom-ns"),
 		WithRunID("test-run"),
 		WithCleanup(false),
@@ -73,6 +80,9 @@ func TestNewWithOptions(t *testing.T) {
 	if v.Commit != "abc1234" {
 		t.Errorf("Commit = %q, want %q", v.Commit, "abc1234")
 	}
+	if v.Kubeconfig != "/path/to/kubeconfig" {
+		t.Errorf("Kubeconfig = %q, want %q", v.Kubeconfig, "/path/to/kubeconfig")
+	}
 	if v.Namespace != "custom-ns" {
 		t.Errorf("Namespace = %q, want %q", v.Namespace, "custom-ns")
 	}
@@ -90,6 +100,84 @@ func TestNewWithOptions(t *testing.T) {
 	}
 }
 
+// TestPrepareClusterPropagatesCustomKubeconfig verifies the run-scoped path
+// reaches cluster client creation without reading a kubeconfig file or
+// contacting Kubernetes. The injected factory fails before any cluster API
+// operation, keeping this regression test hermetic and fail-safe.
+func TestPrepareClusterPropagatesCustomKubeconfig(t *testing.T) {
+	t.Parallel()
+
+	const wantKubeconfig = "/path/to/target-kubeconfig"
+	wantErr := stderrors.New("stop before cluster access")
+	v := New(WithKubeconfig("  " + wantKubeconfig + "  "))
+
+	var gotKubeconfig string
+	v.kubeClientFactory = func(kubeconfig string) (kubernetes.Interface, error) {
+		gotKubeconfig = kubeconfig
+		return nil, wantErr
+	}
+
+	_, err := v.prepareCluster(t.Context(), nil, nil)
+	if !stderrors.Is(err, wantErr) {
+		t.Fatalf("prepareCluster() error = %v, want wrapped injected error", err)
+	}
+	if gotKubeconfig != wantKubeconfig {
+		t.Errorf("kubeconfig = %q, want %q", gotKubeconfig, wantKubeconfig)
+	}
+}
+
+// TestPrepareClusterEmptyKubeconfigUsesDefaultClient verifies that empty input
+// is routed through default discovery without consulting the explicit-path
+// client factory. The environment is cleared so default discovery fails before
+// any cluster access, keeping the test hermetic.
+func TestPrepareClusterEmptyKubeconfigUsesDefaultClient(t *testing.T) {
+	t.Setenv("KUBECONFIG", "")
+	t.Setenv("HOME", t.TempDir())
+	t.Setenv("USERPROFILE", t.TempDir())
+	t.Setenv("KUBERNETES_SERVICE_HOST", "")
+	t.Setenv("KUBERNETES_SERVICE_PORT", "")
+
+	wantFactoryErr := stderrors.New("explicit-path factory called")
+	v := New(WithKubeconfig(" \t "))
+	factoryCalled := false
+	v.kubeClientFactory = func(string) (kubernetes.Interface, error) {
+		factoryCalled = true
+		return nil, wantFactoryErr
+	}
+
+	_, err := v.prepareCluster(t.Context(), nil, nil)
+	if err == nil {
+		t.Fatal("prepareCluster() error = nil, want default discovery error")
+	}
+	if factoryCalled {
+		t.Error("prepareCluster() called explicit-path factory for empty kubeconfig")
+	}
+	if stderrors.Is(err, wantFactoryErr) {
+		t.Errorf("prepareCluster() error = %v, want default discovery error", err)
+	}
+}
+
+// TestPrepareClusterRejectsMissingKubeconfig verifies that a typo in a
+// caller-supplied path is classified as invalid input before Kubernetes client
+// construction can relabel the filesystem error as an internal failure.
+func TestPrepareClusterRejectsMissingKubeconfig(t *testing.T) {
+	t.Parallel()
+
+	kubeconfig := filepath.Join(t.TempDir(), "missing-kubeconfig")
+	v := New(WithKubeconfig(kubeconfig))
+
+	_, err := v.prepareCluster(t.Context(), nil, nil)
+	if err == nil {
+		t.Fatal("prepareCluster() error = nil, want invalid request")
+	}
+	if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) {
+		t.Errorf("prepareCluster() error = %v, want ErrCodeInvalidRequest", err)
+	}
+	if !stderrors.Is(err, fs.ErrNotExist) {
+		t.Errorf("prepareCluster() error = %v, want wrapped fs.ErrNotExist", err)
+	}
+}
+
 func loadEmbeddedCatalog(t *testing.T) *catalog.ValidatorCatalog {
 	t.Helper()
 	cat, err := catalog.LoadWithDataProvider(context.Background(), nil, "", "")