[castai-umbrella] Add e2e pipeline for umbrella (#1355)

oanatmaria · web-flow · commit 78ccb5e2871c · 2026-04-08T15:35:50.000+03:00
diff --git a/.github/workflows/e2e-umbrella.yaml b/.github/workflows/e2e-umbrella.yaml
@@ -0,0 +1,74 @@
+name: Umbrella E2E Tests
+
+on:
+  # Run on PRs where the title contains "castai-umbrella".
+  pull_request:
+    types: [opened, synchronize, reopened, edited]
+  # Run automatically when a castai-umbrella release is published.
+  release:
+    types: [published]
+  # Allow manual runs with optional mode filter.
+  workflow_dispatch:
+    inputs:
+      focus:
+        description: "Ginkgo focus filter (leave empty to run all modes)"
+        required: false
+        default: ""
+
+jobs:
+  e2e:
+    if: >
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'release' && startsWith(github.event.release.tag_name, 'castai-')) ||
+      (github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'castai-umbrella'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: test/e2e/go.mod
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4.3.1
+        with:
+          version: v3.17.0
+
+      - name: Install kind
+        uses: helm/kind-action@v1.12.0
+        with:
+          install_only: true
+
+      - name: Install kubectl
+        uses: azure/setup-kubectl@v4
+
+      - name: Add Helm repos
+        run: |
+          helm repo add castai-helm https://castai.github.io/helm-charts
+          helm repo update castai-helm
+
+      - name: Update umbrella subchart dependencies
+        run: |
+          for sub in kent autoscaler autoscaler-anywhere autoscaler-openshift; do
+            path="charts/castai-umbrella/charts/${sub}"
+            if [ -d "$path" ]; then
+              helm dependency update "$path"
+            fi
+          done
+          helm dependency update charts/castai-umbrella
+
+      - name: Run e2e tests
+        env:
+          CASTAI_API_KEY: ${{ secrets.CASTAI_API_KEY }}
+          API_URL: ${{ vars.CASTAI_API_URL }}
+        run: |
+          FOCUS="${{ github.event.inputs.focus }}"
+          ARGS="-v -timeout 75m"
+          if [ -n "$FOCUS" ]; then
+            ARGS="$ARGS --ginkgo.focus=${FOCUS}"
+          fi
+          cd test/e2e && go test ./... $ARGS
diff --git a/test/e2e/Makefile b/test/e2e/Makefile
@@ -13,7 +13,7 @@
 #   CASTAI_API_KEY=<your-key> make e2e-autoscaler-anywhere
 #   CASTAI_API_KEY=<your-key> make e2e-autoscaler-openshift
 
-.PHONY: e2e e2e-kent e2e-autoscaler e2e-autoscaler-anywhere e2e-autoscaler-openshift e2e-upgrade e2e-uninstall deps tidy
+.PHONY: e2e e2e-kent e2e-autoscaler e2e-autoscaler-workload e2e-autoscaler-anywhere e2e-autoscaler-openshift e2e-upgrade e2e-uninstall deps tidy
 
 # Run all e2e tests (each context creates its own Kind cluster)
 e2e:
@@ -27,6 +27,10 @@ e2e-kent:
 e2e-autoscaler:
 	go test ./... -v -timeout 30m --ginkgo.focus="autoscaler mode"
 
+# Run only the autoscaler workload-autoscaler mode tests
+e2e-autoscaler-workload:
+	go test ./... -v -timeout 30m --ginkgo.focus="autoscaler workload-autoscaler mode"
+
 # Run only the autoscaler-anywhere mode tests
 e2e-autoscaler-anywhere:
 	go test ./... -v -timeout 30m --ginkgo.focus="autoscaler-anywhere mode"
@@ -35,9 +39,9 @@ e2e-autoscaler-anywhere:
 e2e-autoscaler-openshift:
 	go test ./... -v -timeout 30m --ginkgo.focus="autoscaler-openshift mode"
 
-# Run only the upgrade tests
+# Run only the readonly-to-full upgrade tests
 e2e-upgrade:
-	go test ./... -v -timeout 30m --ginkgo.focus="in-place upgrade"
+	go test ./... -v -timeout 30m --ginkgo.focus="autoscaler readonly-to-full upgrade"
 
 # Run only the uninstall tests
 e2e-uninstall:
diff --git a/test/e2e/helpers_test.go b/test/e2e/helpers_test.go
@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"os/exec"
 	"strings"
-	"time"
 
 	. "github.com/onsi/gomega"
 
@@ -26,19 +25,16 @@ func NewUmbrellaHelmHelper(releaseName, namespace, apiURL string) *UmbrellaHelmH
 }
 
 func (h *UmbrellaHelmHelper) InstallKentMode(apiKey string) error {
+	// preflight checks for Karpenter deployment + CRDs which don't exist on Kind.
+	// castai-aws-vpc-cni runs a pre-install hook that patches the aws-node daemonset,
+	// which also doesn't exist on Kind.
 	cmd := exec.Command("helm", "upgrade", "--install", h.releaseName,
 		chartPath,
 		"--namespace", h.namespace,
 		"--create-namespace",
 		"--set", "kent.enabled=true",
-		"--set", "kent.castai-agent.enabled=true",
-		"--set", "kent.castai-kentroller.enabled=true",
-		"--set", "kent.castai-cluster-controller.enabled=true",
-		"--set", "kent.castai-evictor.enabled=true",
-		"--set", "kent.castai-live.enabled=false",
-		"--set", "kent.castai-pod-mutator.enabled=false",
-		"--set", "kent.castai-workload-autoscaler.enabled=false",
-		"--set", "kent.metrics-server.enabled=false",
+		"--set", "kent.preflight.enabled=false",
+		"--set", "kent.castai-live.castai-aws-vpc-cni.enabled=false",
 		"--set", fmt.Sprintf("global.castai.apiKey=%s", apiKey),
 		"--set", fmt.Sprintf("global.castai.apiURL=%s", h.apiURL),
 		"--timeout", defaultHelmTimeout,
@@ -50,19 +46,74 @@ func (h *UmbrellaHelmHelper) InstallKentMode(apiKey string) error {
 	return nil
 }
 
+func (h *UmbrellaHelmHelper) InstallAutoscalerReadonlyMode(apiKey, provider string) error {
+	// readonly tag activates: castai-agent, castai-spot-handler, castai-kvisor, gpu-metrics-exporter.
+	cmd := exec.Command("helm", "upgrade", "--install", h.releaseName,
+		chartPath,
+		"--namespace", h.namespace,
+		"--create-namespace",
+		"--set", "tags.readonly=true",
+		"--set", fmt.Sprintf("global.castai.apiKey=%s", apiKey),
+		"--set", fmt.Sprintf("global.castai.apiURL=%s", h.apiURL),
+		"--set", fmt.Sprintf("global.castai.provider=%s", provider),
+		"--timeout", defaultHelmTimeout,
+	)
+	_, err := utils.Run(cmd)
+	if err != nil {
+		return fmt.Errorf("helm install autoscaler readonly mode failed: %w", err)
+	}
+	return nil
+}
+
+func (h *UmbrellaHelmHelper) UpgradeToFullMode() error {
+	// Swap readonly tag for full. --reuse-values preserves apiKey/apiURL/provider.
+	// readonly → full is not an additive path so both tags must be set explicitly.
+	cmd := exec.Command("helm", "upgrade", h.releaseName,
+		chartPath,
+		"--namespace", h.namespace,
+		"--reuse-values",
+		"--set", "tags.readonly=false",
+		"--set", "tags.full=true",
+		"--timeout", defaultHelmTimeout,
+	)
+	_, err := utils.Run(cmd)
+	if err != nil {
+		return fmt.Errorf("helm upgrade to full mode failed: %w", err)
+	}
+	return nil
+}
+
+func (h *UmbrellaHelmHelper) InstallAutoscalerWorkloadMode(apiKey, provider string) error {
+	// workload-autoscaler tag activates: castai-agent, castai-spot-handler, castai-kvisor,
+	// gpu-metrics-exporter, castai-cluster-controller, castai-evictor, castai-pod-mutator,
+	// castai-workload-autoscaler, castai-workload-autoscaler-exporter.
+	// NOT included: castai-pod-pinner, castai-live (those are node-autoscaler + full only).
+	cmd := exec.Command("helm", "upgrade", "--install", h.releaseName,
+		chartPath,
+		"--namespace", h.namespace,
+		"--create-namespace",
+		"--set", "tags.workload-autoscaler=true",
+		"--set", fmt.Sprintf("global.castai.apiKey=%s", apiKey),
+		"--set", fmt.Sprintf("global.castai.apiURL=%s", h.apiURL),
+		"--set", fmt.Sprintf("global.castai.provider=%s", provider),
+		"--timeout", defaultHelmTimeout,
+	)
+	_, err := utils.Run(cmd)
+	if err != nil {
+		return fmt.Errorf("helm install autoscaler workload mode failed: %w", err)
+	}
+	return nil
+}
+
 func (h *UmbrellaHelmHelper) InstallAutoscalerMode(apiKey, provider string) error {
+	// node-autoscaler tag activates: castai-agent, castai-spot-handler, castai-kvisor,
+	// gpu-metrics-exporter, castai-cluster-controller, castai-evictor, castai-pod-mutator,
+	// castai-pod-pinner, castai-live (see autoscaler/Chart.yaml tag definitions).
 	cmd := exec.Command("helm", "upgrade", "--install", h.releaseName,
 		chartPath,
 		"--namespace", h.namespace,
 		"--create-namespace",
-		"--set", "autoscaler.enabled=true",
-		"--set", "autoscaler.castai-agent.enabled=true",
-		"--set", "autoscaler.castai-cluster-controller.enabled=true",
-		"--set", "autoscaler.castai-evictor.enabled=true",
-		"--set", "autoscaler.castai-live.enabled=false",
-		"--set", "autoscaler.castai-pod-mutator.enabled=false",
-		"--set", "autoscaler.castai-workload-autoscaler.enabled=false",
-		"--set", "autoscaler.castai-kvisor.enabled=true",
+		"--set", "tags.node-autoscaler=true",
 		"--set", fmt.Sprintf("global.castai.apiKey=%s", apiKey),
 		"--set", fmt.Sprintf("global.castai.apiURL=%s", h.apiURL),
 		"--set", fmt.Sprintf("global.castai.provider=%s", provider),
@@ -76,17 +127,14 @@ func (h *UmbrellaHelmHelper) InstallAutoscalerMode(apiKey, provider string) erro
 }
 
 func (h *UmbrellaHelmHelper) InstallAutoscalerAnywhereMode(apiKey, clusterName string) error {
+	// tags.autoscaler-anywhere includes the autoscaler-anywhere sub-chart.
+	// Components within that sub-chart are enabled by default in its values.yaml;
+	// ANYWHERE_CLUSTER_NAME is passed via additionalEnv so the agent can register.
 	cmd := exec.Command("helm", "upgrade", "--install", h.releaseName,
 		chartPath,
 		"--namespace", h.namespace,
 		"--create-namespace",
-		"--set", "autoscaler-anywhere.enabled=true",
-		"--set", "autoscaler-anywhere.castai-agent.enabled=true",
-		"--set", "autoscaler-anywhere.castai-cluster-controller.enabled=true",
-		"--set", "autoscaler-anywhere.castai-evictor.enabled=true",
-		"--set", "autoscaler-anywhere.castai-pod-mutator.enabled=true",
-		"--set", "autoscaler-anywhere.castai-workload-autoscaler.enabled=false",
-		"--set", "autoscaler-anywhere.castai-workload-autoscaler-exporter.enabled=false",
+		"--set", "tags.autoscaler-anywhere=true",
 		"--set", fmt.Sprintf("autoscaler-anywhere.castai-agent.additionalEnv.ANYWHERE_CLUSTER_NAME=%s", clusterName),
 		"--set", fmt.Sprintf("global.castai.apiKey=%s", apiKey),
 		"--set", fmt.Sprintf("global.castai.apiURL=%s", h.apiURL),
@@ -120,9 +168,13 @@ func (h *UmbrellaHelmHelper) InstallAutoscalerOpenshiftMode(apiKey string) error
 }
 
 func (h *UmbrellaHelmHelper) Uninstall() error {
+	// --no-hooks skips pre/post-delete hooks (e.g. webhook teardown calls) that
+	// block when pods are unhealthy. The namespace and Kind cluster are deleted
+	// immediately after, so hook-managed resources are cleaned up anyway.
 	cmd := exec.Command("helm", "uninstall", h.releaseName,
 		"--namespace", h.namespace,
 		"--ignore-not-found",
+		"--no-hooks",
 		"--timeout", defaultHelmTimeout,
 	)
 	_, err := utils.Run(cmd)
@@ -236,12 +288,6 @@ func (n *NamespaceHelper) VerifyExists(g Gomega, namespace string) {
 	g.Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Namespace %s should exist", namespace))
 }
 
-func waitForDeployment(podHelper *PodHelper, name string, timeout time.Duration) {
-	Eventually(func(g Gomega) {
-		podHelper.VerifyDeploymentExists(g, name)
-	}, timeout, 5*time.Second).Should(Succeed())
-}
-
 // VerifyAtLeastOnePodReady checks that the named deployment has at least one
 // available replica according to the deployment status — no label guessing needed.
 func (p *PodHelper) VerifyAtLeastOnePodReady(g Gomega, deploymentName string) {
diff --git a/test/e2e/umbrella_test.go b/test/e2e/umbrella_test.go
