Add a vcluster infrastructure provider for fast task iteration

devops_bench/providers/__init__.py

@@ -19,6 +19,7 @@
 # Import for their registration side effects so the registry is populated.
 from devops_bench.providers import gcp as _gcp  # noqa: F401
 from devops_bench.providers import kind as _kind  # noqa: F401
+from devops_bench.providers import vcluster as _vcluster  # noqa: F401
 from devops_bench.providers.base import PROVIDERS, Provider, ResolveContext
 
 __all__ = ["PROVIDERS", "Provider", "ResolveContext"]

devops_bench/providers/vcluster.py

@@ -0,0 +1,147 @@
+# Copyright 2026 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""vcluster provider: virtual clusters running inside an existing host cluster."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from devops_bench.core import ClusterInfo, ConfigError, get_env, get_logger
+from devops_bench.core.subprocess import run
+from devops_bench.providers.base import PROVIDERS, Provider, ResolveContext
+
+__all__ = ["VclusterProvider"]
+
+_log = get_logger("providers.vcluster")
+
+
+@PROVIDERS.register("vcluster")
+class VclusterProvider(Provider):
+    """Provider for loft-sh vcluster virtual clusters hosted on gke/eks/aks.
+
+    A vcluster is a virtual Kubernetes control plane running as a workload
+    inside an existing host cluster, provisioned in a couple of minutes
+    instead of the ten-plus minutes a real cluster takes. Access to the
+    vcluster itself goes through the kubeconfig the OpenTofu module writes to
+    disk, not through the host cloud's CLI; only reaching the *host* cluster
+    (to run the vcluster's own control plane) may need host cloud
+    credentials.
+    """
+
+    def ensure_account_credentials(self) -> None:
+        """Ensure the host cluster's context is available for kubectl/helm.
+
+        Reads ``VCLUSTER_HOST_CLOUD`` (default ``gke``) to decide how. On
+        ``gke``, if ``VCLUSTER_HOST_CLUSTER`` is set and a project is
+        resolvable from ``GCP_PROJECT_ID``, runs ``gcloud container clusters
+        get-credentials`` for the host cluster so its context exists in
+        kubeconfig; otherwise a no-op, assuming the host context is already
+        present in kubeconfig. On ``eks``/``aks`` this is always a no-op for
+        now: host credential automation (``aws eks update-kubeconfig`` /
+        ``az aks get-credentials``) is not yet implemented, so the host
+        context is assumed to already be present in kubeconfig.
+
+        Raises:
+            ConfigError: If ``VCLUSTER_HOST_CLUSTER`` is set on a ``gke`` host
+                but no location is resolvable from ``VCLUSTER_HOST_LOCATION``
+                or ``GCP_LOCATION``.
+        """
+        host_cloud = get_env("VCLUSTER_HOST_CLOUD") or "gke"
+        if host_cloud != "gke":
+            _log.debug(
+                "vcluster provider: host_cloud=%s, assuming host cluster credentials "
+                "are already present (no automation yet for eks/aks)",
+                host_cloud,
+            )
+            return
+
+        host_cluster = get_env("VCLUSTER_HOST_CLUSTER")
+        project = get_env("GCP_PROJECT_ID")
+        if not host_cluster or not project:
+            _log.debug(
+                "vcluster provider: assuming host cluster context is already in kubeconfig"
+            )
+            return
+
+        location = get_env("VCLUSTER_HOST_LOCATION") or get_env("GCP_LOCATION")
+        if not location:
+            raise ConfigError(
+                "VCLUSTER_HOST_CLUSTER is set but no location is resolvable; "
+                "set VCLUSTER_HOST_LOCATION or GCP_LOCATION"
+            )
+        _log.info("Configuring kubectl for host cluster: %s in %s...", host_cluster, location)
+        run(
+            [
+                "gcloud",
+                "container",
+                "clusters",
+                "get-credentials",
+                host_cluster,
+                "--location",
+                location,
+                "--project",
+                project,
+            ],
+            capture=False,
+        )
+
+    def ensure_cluster_credentials(
+        self, cluster_name: str, location: str, variables: dict[str, Any]
+    ) -> ClusterInfo:
+        """Describe a vcluster; its kubeconfig is already on disk.
+
+        Args:
+            cluster_name: Cluster name from the stack outputs.
+            location: Location from the stack outputs (the host GKE region).
+            variables: OpenTofu input variables the cluster was provisioned with.
+
+        Returns:
+            The cluster's :class:`~devops_bench.core.ClusterInfo`.
+        """
+        project = variables.get("project_id") or get_env("GCP_PROJECT_ID")
+        return ClusterInfo.from_dict(
+            {
+                "name": cluster_name,
+                "location": location,
+                "project": project,
+                "kubeconfig_path": variables.get("kubeconfig_path"),
+            }
+        )
+
+    def resolve_variables(
+        self, ctx: ResolveContext, custom_variables: dict[str, Any]
+    ) -> dict[str, Any]:
+        """Resolve default OpenTofu variables for vcluster stacks.
+
+        Returns:
+            A new mapping with ``project_id``, ``cluster_name``, ``location``,
+            ``host_cloud``, ``host_cluster_name``, ``host_context``, and
+            ``kubeconfig_path`` filled in where not already set.
+        """
+        variables = custom_variables.copy()
+        variables.setdefault("infra_provider", "vcluster")
+        variables.setdefault("project_id", ctx.project_id)
+        variables.setdefault("cluster_name", ctx.cluster_name)
+        variables.setdefault("location", ctx.location)
+        variables.setdefault("host_cloud", get_env("VCLUSTER_HOST_CLOUD") or "gke")
+        variables.setdefault("host_cluster_name", get_env("VCLUSTER_HOST_CLUSTER") or "")
+        host_context = get_env("VCLUSTER_HOST_CONTEXT")
+        if host_context:
+            variables.setdefault("host_context", host_context)
+        cluster_name = variables["cluster_name"]
+        kubeconfig_path = str(Path("~/.kube").expanduser() / f"vcluster-{cluster_name}.yaml")
+        variables.setdefault("kubeconfig_path", kubeconfig_path)
+        return variables

docs/components/infra.md

@@ -30,6 +30,7 @@ A cloud provider supplies credentials and Terraform variable defaults for a stac
 | --- | --- | --- |
 | `GcpProvider` | `gcp` | GKE clusters on Google Cloud |
 | `KindProvider` | `kind` | Local KinD clusters (no cloud identity) |
+| `VclusterProvider` | `vcluster` | loft-sh vcluster virtual clusters inside an existing host cluster (gke/eks/aks) |
 
 Each implements the `Provider` interface (`devops_bench/providers/base.py`):
 
@@ -50,14 +51,79 @@ The env var outranks the config key so a task can pin a default `provider:` whil
 > [!IMPORTANT]
 > There is no default cloud. Any stack that does not deduce to `kind` — including every absolute or external path — **must** name its provider explicitly via `provider:` or `INFRA_PROVIDER`, or `_select_provider` raises a `ConfigError`. Nothing falls back to `gcp`, so a new provider never silently inherits another's defaults. An unknown provider name is likewise a configuration error.
 
+## vcluster: fast virtual clusters on a host cluster
+
+`vcluster` (loft-sh, Helm chart `0.36.1`) runs a virtual Kubernetes control plane as a
+workload inside an existing host cluster, instead of provisioning a new cluster per
+run. Provisioning takes roughly 2-3.5 minutes versus the 10-20 minutes a real GKE
+cluster takes, since the host cluster's nodes and networking already exist. The module
+that provisions it is host-cloud-agnostic: it pre-creates a plain Kubernetes
+`LoadBalancer` Service in front of the vcluster syncer pods and reads back whatever
+external address the host cloud hands out (an IP on GKE and AKS, a hostname on EKS's
+NLBs) before rendering the Helm values, so the proxy cert SANs and the exported
+kubeconfig server always match the real address. There is no cloud-specific resource
+in the module anymore.
+
+**When to use it:** tasks that only need workload-, manifest-, or policy-level
+fidelity — deploying Kubernetes objects, evaluating admission policies, exercising
+controllers and operators, or anything that just needs a real API server to talk to.
+
+**When not to use it:** tasks that depend on node-level fidelity that a virtual
+cluster can't provide — real node pools and machine types, GKE Workload Identity,
+DaemonSets that need to run on real nodes, or anything that inspects the underlying
+node OS or cloud-specific node behavior. Use `gcp` for those.
+
+**Required environment:**
+
+| Variable | Effect |
+| --- | --- |
+| `VCLUSTER_HOST_CLOUD` | Cloud the host cluster runs on: `gke` (default), `eks`, or `aks`. Passed through to the `host_cloud` OpenTofu variable. Only `gke` is implemented end to end today; see "EKS/AKS hosts" below. |
+| `VCLUSTER_HOST_CLUSTER` | Name of the host cluster the vcluster runs inside. On `gke`, if set (with `GCP_PROJECT_ID` resolvable), `ensure_account_credentials()` runs `gcloud container clusters get-credentials` for the host cluster. If unset, the host context is assumed to already be in kubeconfig. |
+| `VCLUSTER_HOST_CONTEXT` | Explicit kube context of the host cluster. On `gke`, overrides the default `gke_<project>_<location>_<host_cluster_name>` naming. Required on `eks`/`aks`, since there is no equivalent naming convention to derive it from. |
+| `GCP_PROJECT_ID` | GCP project of the host cluster (`gke` only). |
+| `GCP_LOCATION` / `VCLUSTER_HOST_LOCATION` | Region of the host cluster (`gke` only). |
+
+A task example:
+
+```yaml
+infrastructure:
+  deployer: "tofu"
+  stack: "prebuilt/vcluster"
+  provider: "vcluster"
+  teardown: true
+```
+
+**Teardown:** the vcluster's Kubernetes namespace is managed directly by OpenTofu
+(not via Helm's `create_namespace`), so `tofu destroy` deletes the namespace and, with
+it, the vcluster StatefulSet's PVC and the pre-created LoadBalancer Service. That means
+a torn-down run leaves no state behind on the host cluster; a plain `helm uninstall`
+would leave the PVC around and let a re-created vcluster resume stale state.
+
+**EKS/AKS hosts:** vcluster itself is supported by loft-sh on any conformant
+Kubernetes cluster, including EKS and AKS, and the `tf/modules/cluster/vcluster`
+module has no GCP-specific resources left, so nothing in the module needs to change to
+run on those hosts. What's missing is on our side: `ensure_account_credentials()` only
+knows how to fetch GKE credentials today (`gcloud container clusters
+get-credentials`); equivalent `aws eks update-kubeconfig` / `az aks get-credentials`
+support hasn't been added yet, so `VCLUSTER_HOST_CONTEXT` must point at an
+already-configured kubeconfig entry when `VCLUSTER_HOST_CLOUD` is `eks` or `aks`. Those
+hosts also haven't been exercised against real EKS/AKS clusters in this repo, so treat
+them as untested until someone runs it.
+
 ## What the Terraform provisions
 
 The OpenTofu stacks live under `tf/`:
 
-- `tf/modules/` — reusable building blocks: `cluster/gke`, `cluster/kind`, and `bastion`.
-- `tf/prebuilt/<stack>/` — standard, ready-to-use stacks, currently `kind` (a local
-  cluster for offline / no-cloud runs). Task-specific stacks build on the modules and
-  ship alongside the tasks that provision them.
+- `tf/modules/` — reusable building blocks: `cluster/gke`, `cluster/kind`,
+  `cluster/vcluster`, and `bastion`.
+- `tf/prebuilt/<stack>/` — standard, ready-to-use stacks: `kind` (a local cluster for
+  offline / no-cloud runs), `vcluster` (a virtual cluster on an existing host cluster), and
+  `minimal` (a provider-agnostic stack that dispatches through `tf/modules/cluster` and
+  flips between `gcp`, `kind`, and `vcluster` via the `infra_provider` variable, which
+  the harness sets from `INFRA_PROVIDER` or a task's `provider:` key; it is named
+  `minimal`, not `minimum`, to avoid colliding with downstream stacks that use that
+  name). Task-specific stacks build on the modules and ship alongside the tasks that
+  provision them.
 
 Every stack root that `TFDeployer` drives must output `cluster_name` and `cluster_location` — that's the contract the deployer reads back.
 
@@ -66,6 +132,7 @@ Every stack root that `TFDeployer` drives must output `cluster_name` and `cluste
 - Always: the `tofu` binary on `PATH`.
 - For GCP stacks: `gcloud`, application-default credentials (ADC), and a project with the GKE and Artifact Registry APIs enabled.
 - For KinD stacks: Docker and the `kind` binary.
+- For vcluster stacks: `kubectl` and `helm` on `PATH`, plus an existing host cluster reachable via kubeconfig (`VCLUSTER_HOST_CLUSTER` or `VCLUSTER_HOST_CONTEXT`). `gcloud` is only needed when `VCLUSTER_HOST_CLOUD` is `gke` (the default).
 
 ## Configuring infra for an eval
 
@@ -75,7 +142,7 @@ Infrastructure is declared in the `infrastructure:` block of a task's `task.yaml
 | --- | --- |
 | `deployer` | `tofu` or `noop`. |
 | `stack` | Stack name under `tf/` (e.g. `prebuilt/kind`) or an absolute path. |
-| `provider` | Optional. `gcp` or `kind`. Omit to let it be deduced (in-repo stacks only). |
+| `provider` | Optional. `gcp`, `kind`, or `vcluster`. Omit to let it be deduced (in-repo stacks only). |
 | `teardown` | Whether to destroy infra after the run. Defaults to `true`. |
 | `variables` | A map passed straight to `tofu` as `-var key=value` flags. |
 
@@ -99,7 +166,7 @@ infrastructure:
   deployer: "noop"
 ```
 
-The provider fills in sensible defaults for whatever you leave out. For GCP that means `project_id`, `cluster_name`, and `location` (plus `namespace` when `NAMESPACE` is set); for KinD it means `cluster_name`, `location` (`local`), and `kubeconfig_path`. Anything you put in `variables` always wins over the defaults.
+The provider fills in sensible defaults for whatever you leave out. For GCP that means `project_id`, `cluster_name`, and `location` (plus `namespace` when `NAMESPACE` is set); for KinD it means `cluster_name`, `location` (`local`), and `kubeconfig_path`; for vcluster it means `project_id`, `cluster_name`, `location`, `host_cloud`, `host_cluster_name`, `host_context`, and `kubeconfig_path`. Anything you put in `variables` always wins over the defaults.
 
 **Environment variables that affect infra:**
 
@@ -111,6 +178,10 @@ The provider fills in sensible defaults for whatever you leave out. For GCP that
 | `GCP_LOCATION` | Default region/zone (falls back to `us-central1-a`). |
 | `NAMESPACE` | Passed through to GCP stacks as the `namespace` variable. |
 | `KUBECONFIG` | Kubeconfig path used by KinD and by no-infra runs. |
+| `VCLUSTER_HOST_CLOUD` | Cloud the vcluster host cluster runs on: `gke` (default), `eks`, or `aks`. |
+| `VCLUSTER_HOST_CLUSTER` | Name of the host cluster a vcluster runs inside; on `gke`, also used to fetch host credentials via `gcloud`. |
+| `VCLUSTER_HOST_CONTEXT` | Explicit kube context of the host cluster, overriding the derived `gke_<project>_<location>_<host_cluster_name>` name (`gke` only). Required when `VCLUSTER_HOST_CLOUD` is `eks` or `aks`. |
+| `VCLUSTER_HOST_LOCATION` | Region of the host GKE cluster (falls back to `GCP_LOCATION`, `gke` only). |
 
 The `--project` and `--cluster` CLI flags supply the project and cluster name for a run, feeding the same defaults the providers resolve from.