Remove experimental guard from downstream resource copy (#5207)

thardeck · web-flow · commit ffa11a444b45 · 2026-05-29T16:17:01.000Z
* Remove experimental guard from downstream resource copy

Drop the EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM environment variable
and always enable the downstream resource copy feature. Remove the
env var from agent manifests, RBAC bootstrap, CI scripts, chart
values, and all associated test setup. The configmaps ClusterRole
rule is now unconditional.

* Collapse secrets and configmaps into single RBAC rule
diff --git a/.github/scripts/deploy-fleet.sh b/.github/scripts/deploy-fleet.sh
@@ -71,8 +71,6 @@ eventually helm upgrade --install fleet charts/fleet \
   --set imagescan.enabled=true \
   --set-string extraEnv[0].name=EXPERIMENTAL_SCHEDULES \
   --set-string extraEnv[0].value=true \
-  --set-string extraEnv[1].name=EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM \
-  --set-string extraEnv[1].value=true \
   --set debug=true --set debugLevel=1
 
 # wait for controller and agent rollout
diff --git a/.github/scripts/setup-rancher.sh b/.github/scripts/setup-rancher.sh
@@ -27,9 +27,7 @@ helm upgrade rancher rancher-latest/rancher $args \
   --set hostname="$public_hostname" \
   --set bootstrapPassword=admin \
   --set "extraEnv[0].name=CATTLE_SERVER_URL" \
-  --set "extraEnv[0].value=https://$public_hostname" \
-  --set-string "fleet.extraEnv[0].name=EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM" \
-  --set-string "fleet.extraEnv[0].value=true"
+  --set "extraEnv[0].value=https://$public_hostname"
 
 # wait for deployment of rancher
 kubectl -n cattle-system rollout status deploy/rancher
diff --git a/.github/workflows/e2e-test-fleet-in-rancher.yml b/.github/workflows/e2e-test-fleet-in-rancher.yml
@@ -230,9 +230,7 @@ jobs:
             --set "extraEnv[2].name=CATTLE_FLEET_VERSION" \
             --set "extraEnv[2].value=${fleet_version}" \
             --set "extraEnv[3].name=CATTLE_SERVER_URL" \
-            --set "extraEnv[3].value=https://$ip.sslip.io" \
-            --set-string fleet.extraEnv[0].name=EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM \
-            --set-string fleet.extraEnv[0].value=true
+            --set "extraEnv[3].value=https://$ip.sslip.io"
 
           # wait for deployment of rancher
           kubectl -n cattle-system rollout status deploy/rancher
diff --git a/charts/fleet/ci/debug-values.yaml b/charts/fleet/ci/debug-values.yaml
@@ -66,7 +66,5 @@ shards:
       kubernetes.io/hostname: k3d-upstream-server-2
 
 extraEnv:
-  - name: EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM
-    value: "true"
   - name: EXPERIMENTAL_SCHEDULES
     value: "true"
diff --git a/charts/fleet/ci/nobootstrap-values.yaml b/charts/fleet/ci/nobootstrap-values.yaml
@@ -65,7 +65,5 @@ shards:
       kubernetes.io/hostname: k3d-upstream-server-2
 
 extraEnv:
-  - name: EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM
-    value: "true"
   - name: EXPERIMENTAL_SCHEDULES
     value: "true"
diff --git a/charts/fleet/ci/nodebug-values.yaml b/charts/fleet/ci/nodebug-values.yaml
@@ -65,7 +65,5 @@ shards:
       kubernetes.io/hostname: k3d-upstream-server-2
 
 extraEnv:
-  - name: EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM
-    value: "true"
   - name: EXPERIMENTAL_SCHEDULES
     value: "true"
diff --git a/charts/fleet/ci/nogitops-values.yaml b/charts/fleet/ci/nogitops-values.yaml
@@ -65,7 +65,5 @@ shards:
       kubernetes.io/hostname: k3d-upstream-server-2
 
 extraEnv:
-  - name: EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM
-    value: "true"
   - name: EXPERIMENTAL_SCHEDULES
     value: "true"
diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml
@@ -195,8 +195,6 @@ agent:
 # extraEnv:
 # - name: OCI_STORAGE
 #   value: "false"
-# - name: EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM
-#   value: "false"
 
 # shards:
 #   - id: shard0
diff --git a/dev/setup-fleet b/dev/setup-fleet
@@ -49,8 +49,6 @@ helm -n cattle-fleet-system upgrade --install --create-namespace --wait --reset-
   --set imagescan.enabled=true \
   --set-string extraEnv[0].name=EXPERIMENTAL_SCHEDULES \
   --set-string extraEnv[0].value=true \
-  --set-string extraEnv[1].name=EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM \
-  --set-string extraEnv[1].value=true \
   --set debug=true --set debugLevel=1 fleet charts/fleet
 
 # wait for controller and agent rollout
diff --git a/integrationtests/controller/bundle/bundle_downstream_resources_test.go b/integrationtests/controller/bundle/bundle_downstream_resources_test.go
@@ -1,14 +1,12 @@
 package bundle
 
 import (
-	"os"
 	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/rancher/fleet/integrationtests/utils"
-	"github.com/rancher/fleet/internal/experimental"
 	"github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 
 	corev1 "k8s.io/api/core/v1"
@@ -28,11 +26,7 @@ var _ = Describe("Bundle target DownstreamResources", Ordered, func() {
 
 		createClustersAndClusterGroups()
 
-		// Enable the experimental downstream copy feature.
-		Expect(os.Setenv(experimental.CopyResourcesDownstreamFlag, "true")).To(Succeed())
-
 		DeferCleanup(func() {
-			os.Unsetenv(experimental.CopyResourcesDownstreamFlag)
 			Expect(k8sClient.Delete(ctx, ns)).ToNot(HaveOccurred())
 		})
 	})
diff --git a/internal/cmd/agent/controller/bundledeployment_controller.go b/internal/cmd/agent/controller/bundledeployment_controller.go
@@ -12,7 +12,6 @@ import (
 	"github.com/rancher/fleet/internal/cmd/agent/deployer/cleanup"
 	"github.com/rancher/fleet/internal/cmd/agent/deployer/driftdetect"
 	"github.com/rancher/fleet/internal/cmd/agent/deployer/monitor"
-	"github.com/rancher/fleet/internal/experimental"
 	"github.com/rancher/fleet/internal/helmvalues"
 	"github.com/rancher/fleet/internal/namespaces"
 	fleetv1 "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
@@ -366,10 +365,6 @@ func (r *BundleDeploymentReconciler) copyResourcesFromUpstream(
 	bd *fleetv1.BundleDeployment,
 	logger logr.Logger,
 ) (bool, error) {
-	if !experimental.CopyResourcesDownstreamEnabled() {
-		return false, nil
-	}
-
 	if len(bd.Spec.Options.DownstreamResources) == 0 {
 		return false, nil
 	}
diff --git a/internal/cmd/controller/agentmanagement/agent/manifest.go b/internal/cmd/controller/agentmanagement/agent/manifest.go
@@ -10,7 +10,6 @@ import (
 
 	"github.com/rancher/fleet/internal/cmd"
 	"github.com/rancher/fleet/internal/config"
-	"github.com/rancher/fleet/internal/experimental"
 	"github.com/rancher/fleet/internal/names"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -197,7 +196,6 @@ func agentApp(namespace string, agentScope string, opts ManifestOptions) *appsv1
 								{Name: "CATTLE_ELECTION_LEASE_DURATION", Value: opts.LeaseDuration.String()},
 								{Name: "CATTLE_ELECTION_RETRY_PERIOD", Value: opts.RetryPeriod.String()},
 								{Name: "CATTLE_ELECTION_RENEW_DEADLINE", Value: opts.RenewDeadline.String()},
-								{Name: "EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM", Value: strconv.FormatBool(experimental.CopyResourcesDownstreamEnabled())},
 							},
 							Command: []string{
 								"fleetagent",
diff --git a/internal/cmd/controller/agentmanagement/controllers/resources/data.go b/internal/cmd/controller/agentmanagement/controllers/resources/data.go
@@ -1,7 +1,6 @@
 package resources
 
 import (
-	"github.com/rancher/fleet/internal/experimental"
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 
 	"github.com/rancher/wrangler/v3/pkg/apply"
@@ -32,18 +31,10 @@ func ApplyBootstrapResources(systemNamespace, systemRegistrationNamespace string
 		{
 			Verbs:     []string{"get"},
 			APIGroups: []string{""},
-			Resources: []string{"secrets"},
+			Resources: []string{"secrets", "configmaps"},
 		},
 	}
 
-	if experimental.CopyResourcesDownstreamEnabled() {
-		rules = append(rules, rbacv1.PolicyRule{
-			Verbs:     []string{"get"},
-			APIGroups: []string{""},
-			Resources: []string{"configmaps"},
-		})
-	}
-
 	return apply.ApplyObjects(
 		// used by request-* service accounts from agents
 		&rbacv1.ClusterRole{
diff --git a/internal/cmd/controller/reconciler/bundle_controller.go b/internal/cmd/controller/reconciler/bundle_controller.go
@@ -20,7 +20,6 @@ import (
 	"github.com/rancher/fleet/internal/cmd/controller/summary"
 	"github.com/rancher/fleet/internal/cmd/controller/target"
 	"github.com/rancher/fleet/internal/config"
-	"github.com/rancher/fleet/internal/experimental"
 	"github.com/rancher/fleet/internal/helmvalues"
 	"github.com/rancher/fleet/internal/manifest"
 	"github.com/rancher/fleet/internal/metrics"
@@ -815,10 +814,6 @@ func (r *BundleReconciler) updateErrorStatus(
 }
 
 func (r *BundleReconciler) handleDownstreamObjects(ctx context.Context, bundle *fleet.Bundle, bd *fleet.BundleDeployment) error {
-	if !experimental.CopyResourcesDownstreamEnabled() {
-		return nil
-	}
-
 	// Track if any resources were created or updated
 	resourcesUpdated := false
 
diff --git a/internal/cmd/controller/reconciler/bundle_controller_test.go b/internal/cmd/controller/reconciler/bundle_controller_test.go
@@ -688,9 +688,6 @@ func TestReconcile_OCIReferenceSecretResolutionError(t *testing.T) {
 }
 
 func TestReconcile_DownstreamObjectsHandlingError(t *testing.T) {
-	envVar := "EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM"
-	t.Setenv(envVar, "true")
-
 	cases := []struct {
 		name                        string
 		downstreamResources         []fleetv1.DownstreamResource
@@ -1244,9 +1241,6 @@ func expectGetWithFinalizer(mockCli *mocks.MockK8sClient, bundle fleetv1.Bundle)
 }
 
 func TestReconcile_DownstreamResourcesGeneration_Increment(t *testing.T) {
-	envVar := "EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM"
-	t.Setenv(envVar, "true")
-
 	testCases := []struct {
 		name                     string
 		downstreamResources      []fleetv1.DownstreamResource
@@ -1555,130 +1549,6 @@ func TestReconcile_DownstreamResourcesGeneration_Increment(t *testing.T) {
 	}
 }
 
-func TestReconcile_DownstreamResources_FeatureDisabled(t *testing.T) {
-	envVar := "EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM"
-	// Explicitly disable the feature
-	t.Setenv(envVar, "false")
-
-	mockCtrl := gomock.NewController(t)
-	defer mockCtrl.Finish()
-
-	scheme := runtime.NewScheme()
-	utilruntime.Must(corev1.AddToScheme(scheme))
-	utilruntime.Must(fleetv1.AddToScheme(scheme))
-
-	bundle := fleetv1.Bundle{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test-bundle",
-			Namespace: "default",
-		},
-		Spec: fleetv1.BundleSpec{
-			BundleDeploymentOptions: fleetv1.BundleDeploymentOptions{
-				DownstreamResources: []fleetv1.DownstreamResource{
-					{Kind: "Secret", Name: "my-secret"},
-					{Kind: "ConfigMap", Name: "my-config"},
-				},
-			},
-		},
-	}
-
-	existingBundleDeployment := &fleetv1.BundleDeployment{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test-bd",
-			Namespace: "cluster-ns",
-			UID:       "test-uid",
-		},
-		Spec: fleetv1.BundleDeploymentSpec{
-			DeploymentID:                  "test-deployment",
-			DownstreamResourcesGeneration: 5, // Has a non-zero value
-		},
-	}
-
-	namespacedName := types.NamespacedName{Name: bundle.Name, Namespace: bundle.Namespace}
-
-	mockClient := mocks.NewMockK8sClient(mockCtrl)
-	expectGetWithFinalizer(mockClient, bundle)
-
-	// Options secret deletion
-	mockClient.EXPECT().Delete(gomock.Any(), gomock.AssignableToTypeOf(&corev1.Secret{}), gomock.Any()).
-		Return(&k8serrors.StatusError{ErrStatus: metav1.Status{Code: http.StatusNotFound}})
-
-	matchedTargets := []*target.Target{
-		{
-			Bundle: &bundle,
-			Cluster: &fleetv1.Cluster{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "fleet-default",
-					Name:      "test-cluster",
-				},
-				Status: fleetv1.ClusterStatus{
-					Namespace: "cluster-ns",
-				},
-			},
-			Deployment:   existingBundleDeployment,
-			DeploymentID: "test-deployment",
-		},
-	}
-
-	targetBuilderMock := mocks.NewMockTargetBuilder(mockCtrl)
-	targetBuilderMock.EXPECT().Targets(gomock.Any(), gomock.Any(), gomock.Any()).Return(matchedTargets, false, nil)
-
-	storeMock := mocks.NewMockStore(mockCtrl)
-	storeMock.EXPECT().Store(gomock.Any(), gomock.Any()).Return(nil)
-
-	// List BundleDeployments for cleanup
-	mockClient.EXPECT().List(gomock.Any(), gomock.AssignableToTypeOf(&fleetv1.BundleDeploymentList{}), gomock.Any()).
-		DoAndReturn(func(ctx context.Context, list *fleetv1.BundleDeploymentList, opts ...any) error {
-			list.Items = []fleetv1.BundleDeployment{*existingBundleDeployment}
-			return nil
-		})
-
-	// BundleDeployment get (for CreateOrUpdate)
-	mockClient.EXPECT().Get(gomock.Any(), types.NamespacedName{Namespace: "cluster-ns", Name: "test-bd"}, gomock.AssignableToTypeOf(&fleetv1.BundleDeployment{}), gomock.Any()).
-		DoAndReturn(func(ctx context.Context, key types.NamespacedName, bd *fleetv1.BundleDeployment, opts ...any) error {
-			*bd = *existingBundleDeployment
-			return nil
-		})
-
-	// BundleDeployment update - when feature is disabled, generation stays unchanged
-	// because handleDownstreamObjects modifies BD in memory but doesn't persist it
-	mockClient.EXPECT().Update(gomock.Any(), gomock.AssignableToTypeOf(&fleetv1.BundleDeployment{}), gomock.Any()).
-		DoAndReturn(func(ctx context.Context, obj client.Object, opts ...any) error {
-			bd := obj.(*fleetv1.BundleDeployment)
-			// Feature is disabled, so resources are not cloned and generation stays as-is (5)
-			// The handleDownstreamObjects sets it to 0 in memory but doesn't persist that change
-			if bd.Spec.DownstreamResourcesGeneration != 5 {
-				t.Errorf("Expected generation to remain unchanged at 5 when feature is disabled, got %d", bd.Spec.DownstreamResourcesGeneration)
-			}
-			return nil
-		})
-
-	// No resource cloning calls expected since feature is disabled
-
-	// Status update
-	statusClient := mocks.NewMockStatusWriter(mockCtrl)
-	mockClient.EXPECT().Status().Return(statusClient).Times(1)
-	statusClient.EXPECT().Patch(gomock.Any(), gomock.AssignableToTypeOf(&fleetv1.Bundle{}), gomock.Any()).
-		Return(nil)
-
-	recorderMock := mocks.NewMockEventRecorder(mockCtrl)
-
-	r := reconciler.BundleReconciler{
-		Client:   mockClient,
-		Scheme:   scheme,
-		Recorder: recorderMock,
-		Builder:  targetBuilderMock,
-		Store:    storeMock,
-	}
-
-	ctx := context.TODO()
-	_, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: namespacedName})
-
-	if err != nil {
-		t.Errorf("unexpected error: %v", err)
-	}
-}
-
 // secretNotFoundReader wraps a client.Reader and returns NotFound for a
 // specific secret name/namespace, simulating transient etcd/API-server
 // pressure where the options secret cannot be read inside Targets().
diff --git a/internal/experimental/experimental.go b/internal/experimental/experimental.go
@@ -6,17 +6,9 @@ import (
 )
 
 const (
-	CopyResourcesDownstreamFlag = "EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM"
-	SchedulesFlag               = "EXPERIMENTAL_SCHEDULES"
+	SchedulesFlag = "EXPERIMENTAL_SCHEDULES"
 )
 
-// CopyResourcesDownstreamEnabled returns true if the EXPERIMENTAL_COPY_RESOURCES_DOWNSTREAM env variable
-// is set to true; returns false otherwise.
-func CopyResourcesDownstreamEnabled() bool {
-	value, err := strconv.ParseBool(os.Getenv(CopyResourcesDownstreamFlag))
-	return err == nil && value
-}
-
 // SchedulesEnabled returns true if the EXPERIMENTAL_SCHEDULES env variable is set to true
 // returns false otherwise
 func SchedulesEnabled() bool {
diff --git a/internal/helmdeployer/delete.go b/internal/helmdeployer/delete.go
@@ -13,7 +13,6 @@ import (
 	"helm.sh/helm/v4/pkg/storage/driver"
 
 	"github.com/rancher/fleet/internal/cmd/agent/deployer/kv"
-	"github.com/rancher/fleet/internal/experimental"
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 
 	corev1 "k8s.io/api/core/v1"
@@ -180,10 +179,6 @@ func deleteHistory(cfg *action.Configuration, logger logr.Logger, bundleID strin
 // deleteResourcesCopiedFromUpstream deletes resources referenced through a bundle's `DownstreamResources`
 // field, and copied from downstream.
 func deleteResourcesCopiedFromUpstream(ctx context.Context, c client.Client, bdName string) error {
-	if !experimental.CopyResourcesDownstreamEnabled() {
-		return nil
-	}
-
 	var merr []error
 
 	// No information is available about a deleted bundle deployment beside its name and namespace;
diff --git a/internal/helmdeployer/install.go b/internal/helmdeployer/install.go
@@ -18,7 +18,6 @@ import (
 	releasev1 "helm.sh/helm/v4/pkg/release/v1"
 	"helm.sh/helm/v4/pkg/storage/driver"
 
-	"github.com/rancher/fleet/internal/experimental"
 	"github.com/rancher/fleet/internal/helmdeployer/render"
 	"github.com/rancher/fleet/internal/manifest"
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
@@ -658,9 +657,6 @@ func getDryRunConfig(chart *chartv2.Chart, dryRun bool) dryRunConfig {
 // If not found, returns false.
 func isInDownstreamResources(resourceName, kind string, options fleet.BundleDeploymentOptions) bool {
 	kind = strings.ToLower(kind)
-	if !experimental.CopyResourcesDownstreamEnabled() {
-		return false
-	}
 
 	for _, dr := range options.DownstreamResources {
 		if dr.Name == resourceName && strings.ToLower(dr.Kind) == kind {
diff --git a/internal/helmdeployer/install_test.go b/internal/helmdeployer/install_test.go
