e2e: add nvmeof driver testing support with k8s-e2e-external-storage

scripts/deploy-ceph-csi-operator.sh

@@ -26,6 +26,7 @@ ENCRYPTION_CONFIGMAP_NAME="ceph-csi-encryption-kms-config"
 RBD_DRIVER_NAME="rbd.csi.ceph.com"
 CEPHFS_DRIVER_NAME="cephfs.csi.ceph.com"
 NFS_DRIVER_NAME="nfs.csi.ceph.com"
+NVMEOF_DRIVER_NAME="nvmeof.csi.ceph.com"
 
 # k8s csi sidecar image
 K8S_IMAGE_REPO=${K8S_IMAGE_REPO:-"registry.k8s.io/sig-storage"}
@@ -114,6 +115,7 @@ function deploy_operator() {
     generate_driver "${RBD_DRIVER_NAME}"
     generate_driver "${CEPHFS_DRIVER_NAME}"
     generate_driver "${NFS_DRIVER_NAME}"
+    generate_driver "${NVMEOF_DRIVER_NAME}"
 
     # Display the contents of the generated files for debugging
     for file in "${TEMP_DIR}"/*; do
@@ -129,6 +131,7 @@ function cleanup() {
     generate_driver "${RBD_DRIVER_NAME}"
     generate_driver "${CEPHFS_DRIVER_NAME}"
     generate_driver "${NFS_DRIVER_NAME}"
+    generate_driver "${NVMEOF_DRIVER_NAME}"
     generate_operator_config
 
     # Delete all the generated files at once

scripts/k8s-storage/README.md

@@ -17,4 +17,33 @@ The Ceph-CSI Configuration from the `ceph-csi-config` ConfigMap is created with
 referenced in the StorageClasses and contains the connection details for the
 Ceph cluster.
 
+## Driver-specific Configuration
+
+### NVMeoF Driver
+
+The NVMeoF driver requires additional configuration parameters that can be set
+via environment variables before running `create-storageclasses.sh`. All
+parameters are auto-detected from the `rook-ceph-nvmeof` service and pod if not
+set:
+
+- `GATEWAY_ADDRESS`: NVMeoF gateway IP address (auto-detected from the
+  `rook-ceph-nvmeof` service `clusterIP`)
+- `SHORT_HOSTNAME`: short hostname of the gateway (auto-detected from the
+  `rook-ceph-nvmeof` service name)
+- `POD_ADDRESS`: IP address of the NVMeoF gateway pod (auto-detected from the
+  `rook-ceph-nvmeof` pod); needed because the gateway is deployed in the
+  `rook-ceph` namespace and its short hostname cannot be resolved from the
+  ceph-csi testing namespace
+- `LISTENERS`: JSON array of listener configurations (auto-generated from
+  `POD_ADDRESS`, port `4420`, and `SHORT_HOSTNAME` if not set)
+
+Example:
+
+```bash
+export GATEWAY_ADDRESS=10.242.64.32
+export POD_ADDRESS=10.242.64.32
+export SHORT_HOSTNAME=rook-ceph-nvmeof
+./create-storageclasses.sh
+```
+
 [1]: https://github.com/kubernetes/kubernetes/tree/master/test/e2e/storage/external

scripts/k8s-storage/create-storageclasses.sh

@@ -20,8 +20,58 @@ WORKDIR=$(dirname "${0}")
 TOOLBOX_POD=$(kubectl -n rook-ceph get pods --no-headers -l app=rook-ceph-tools -o=jsonpath='{.items[0].metadata.name}')
 FS_ID=$(kubectl -n rook-ceph exec "${TOOLBOX_POD}" -- ceph fsid)
 
+# NVMeoF-specific parameters (can be overridden via environment variables)
+GATEWAY_ADDRESS=${GATEWAY_ADDRESS:-""}
+LISTENERS=${LISTENERS:-""}
+SHORT_HOSTNAME=${SHORT_HOSTNAME:-""}
+# FIXME: Only pass "hostname" in the LISTENERS, no "address" and "port".
+# POD_ADDRESS is needed as the nvmeof-gw is deployed in rook-ceph, and ceph-csi
+# in a dedicated testing namespace. Resolving the short-hostname of the gateway
+# is not possible from outside the rook-ceph namespace. The gateway is
+# configured with a host-id as the short-hostname, which needs to match what is
+# passed in the LISTENERS.
+POD_ADDRESS=${POD_ADDRESS:-""}
+
+# Auto-detect gateway address and listeners from rook-ceph-nvmeof service if not set
+if [ -z "${GATEWAY_ADDRESS}" ]; then
+	GATEWAY_ADDRESS=$(kubectl -n rook-ceph get service -l app=rook-ceph-nvmeof -o=jsonpath='{.items[0].spec.clusterIP}' 2>/dev/null || echo "")
+fi
+
+if [ -z "${SHORT_HOSTNAME}" ]; then
+	SHORT_HOSTNAME=$(kubectl -n rook-ceph get service -l app=rook-ceph-nvmeof -o=jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
+fi
+
+if [ -z "${POD_ADDRESS}" ]; then
+	POD_ADDRESS=$(kubectl -n rook-ceph get pod -l app=rook-ceph-nvmeof -o=jsonpath='{.items[0].status.podIP}' 2>/dev/null || echo "")
+fi
+
+if [ -z "${LISTENERS}" ] && [ -n "${POD_ADDRESS}" ] && [ -n "${SHORT_HOSTNAME}" ]; then
+	# Create a simple listener config with the gateway pod IP address and hostname
+	LISTENERS='[{"address": "'"${POD_ADDRESS}"'", "port": 4420, "hostname": "'"${SHORT_HOSTNAME}"'"}]'
+fi
+
 for sc in "${WORKDIR}"/sc-*.yaml.in
 do
-	sed "s/@@CLUSTER_ID@@/${FS_ID}/" "${sc}" |
-		kubectl create -f -
+	# Start with CLUSTER_ID replacement
+	SC_CONTENT=$(sed "s/@@CLUSTER_ID@@/${FS_ID}/" "${sc}")
+
+	# For nvmeof, also replace nvmeof-specific parameters
+	if echo "${sc}" | grep -q "nvmeof"; then
+		if [ -z "${GATEWAY_ADDRESS}" ]; then
+			echo "Warning: GATEWAY_ADDRESS not set and could not auto-detect rook-ceph-nvmeof service"
+			echo "Skipping ${sc}"
+			continue
+		fi
+		if [ -z "${LISTENERS}" ]; then
+			echo "Warning: LISTENERS not set and could not auto-detect rook-ceph-nvmeof hostnames"
+			echo "Skipping ${sc}"
+			continue
+		fi
+
+		SC_CONTENT=$(echo "${SC_CONTENT}" | sed \
+			-e "s|@@GATEWAY_ADDRESS@@|${GATEWAY_ADDRESS}|g" \
+			-e "s|@@LISTENERS@@|${LISTENERS}|g")
+	fi
+
+	echo "${SC_CONTENT}" | kubectl create -f -
 done

scripts/k8s-storage/driver-nvmeof.yaml

@@ -0,0 +1,87 @@
+---
+ShortName: cephcsi-nvmeof-test
+Timeouts:
+  ClaimProvision: 10m
+
+StorageClass:
+  FromExistingClassName: k8s-storage-e2e-nvmeof
+  # FromFile: sc-nvmeof.yaml
+
+SnapshotClass:
+  # Must be set to enable snapshotting tests
+  FromExistingClassName: k8s-storage-e2e-nvmeof
+
+DriverInfo:
+  # Internal name of the driver, display name in the test case and test objects
+  Name: nvmeof.csi.ceph.com
+
+  # The range of disk size supported by this driver
+  SupportedSizeRange:
+    Min: 1Gi
+    Max: 16Ti
+
+  # Map of strings for supported FS types
+  SupportedFsType:
+    ext4: {}
+    xfs: {}
+
+  # Map of strings for supported mount options
+  SupportedMountOption:
+    rw: {}
+
+  # Map of strings for required mount options
+  RequiredMountOption:
+    rw: {}
+
+  # Optional list of access modes required for provisioning. Default is RWO
+  # RequiredAccessModes:
+
+  # Map that represents the capabilities the driver supports
+  Capabilities:
+    # Data is persisted across pod restarts
+    persistence: true
+
+    # Volume ownership via fsGroup
+    fsGroup: false
+
+    # Raw block mode
+    block: true
+
+    # Exec a file in the volume
+    exec: true
+
+    # Support for volume limits
+    volumeLimits: false
+
+    # Support for volume expansion in controllers
+    controllerExpansion: true
+
+    # Support for volume expansion in nodes
+    nodeExpansion: true
+
+    # supports offline volume expansion
+    offlineExpansion: false
+
+    # Support volume that can run on single node only (like hostpath)
+    singleNodeVolume: false
+
+    # Support ReadWriteMany access modes (block mode only)
+    RWX: false
+
+    # Support topology
+    topology: false
+
+    # Support populate data from snapshot
+    snapshotDataSource: true
+
+    # Support populated data from PVC
+    pvcDataSource: false
+
+    # multiple pods on a node can use the same volume concurrently
+    multipods: true
+
+    # support ReadWriteOncePod access mode
+    readWriteOncePod: true
+
+    # supports ROX AccessMode in PVC for PVC with Snapshot DataSource
+    capReadOnlyMany: true

scripts/k8s-storage/sc-nvmeof.yaml.in

@@ -0,0 +1,29 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: k8s-storage-e2e-nvmeof
+provisioner: nvmeof.csi.ceph.com
+parameters:
+   clusterID: @@CLUSTER_ID@@
+   pool: replicapool
+   imageFeatures: layering
+   # FIXME: use nvmeof specific secrets, but they don't exist yet?
+   csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
+   csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+   csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
+   csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+   csi.storage.k8s.io/controller-publish-secret-name: rook-csi-rbd-provisioner
+   csi.storage.k8s.io/controller-publish-secret-namespace: rook-ceph
+   csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
+   csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+   csi.storage.k8s.io/fstype: ext4
+   subsystemNQN: nqn.2025-08.io.ceph:k8s-ceph-csi-e2e
+   nvmeofGatewayAddress: @@GATEWAY_ADDRESS@@
+   nvmeofGatewayPort: "5500"
+   listeners: |
+      @@LISTENERS@@
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+mountOptions:
+   - discard

scripts/k8s-storage/volumesnapshotclass-nvmeof.yaml.in

@@ -0,0 +1,12 @@
+---
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+  name: k8s-storage-e2e-nvmeof
+driver: nvmeof.csi.ceph.com
+parameters:
+  clusterID: @@CLUSTER_ID@@
+  # FIXME: use nvmeof specific secrets, but they don't exist yet?
+  csi.storage.k8s.io/snapshotter-secret-name: rook-csi-rbd-provisioner
+  csi.storage.k8s.io/snapshotter-secret-namespace: rook-ceph
+deletionPolicy: Delete

scripts/rook.sh

@@ -56,6 +56,10 @@ function deploy_rook() {
 		kubectl_retry create -f "${TEMP_DIR}/cluster-test.yaml"
 	fi
 
+	curl -o "${TEMP_DIR}/nvmeof-test.yaml" "${ROOK_URL}/nvmeof-test.yaml"
+	sed -i 's|pool: nvmeof|pool: replicapool|g' "${TEMP_DIR}/nvmeof-test.yaml"
+	kubectl_retry create -f "${TEMP_DIR}/nvmeof-test.yaml"
+
 	rm -rf "${TEMP_DIR}"
 
 	kubectl_retry create -f "${ROOK_URL}/toolbox.yaml"
@@ -69,8 +73,8 @@ function deploy_rook() {
 		check_ceph_cluster_health
 	fi
 
-	# Make sure Ceph Mgr is running
-	check_ceph_mgr
+	# Make sure the Ceph NVMe-oF gateway is running
+	check_nvmeof_gw
 
 	# Check if CephFileSystem is empty
 	if ! kubectl_retry -n rook-ceph get cephfilesystems -oyaml | grep 'items: \[\]' &>/dev/null; then
@@ -85,6 +89,7 @@ function deploy_rook() {
 
 function teardown_rook() {
 	create_or_delete_subvolumegroup "delete"
+	kubectl delete -f "${ROOK_URL}/nvmeof-test.yaml"
 	kubectl delete -f "${ROOK_URL}/pool-test.yaml"
 	kubectl delete -f "${ROOK_URL}/filesystem-test.yaml"
 	kubectl delete -f "${ROOK_URL}/toolbox.yaml"
@@ -247,6 +252,21 @@ function check_rbd_stat() {
 	echo ""
 }
 
+function check_nvmeof_gw() {
+	for ((retry = 0; retry <= ROOK_DEPLOY_TIMEOUT; retry = retry + 5)); do
+		echo "Waiting for Ceph NVMe-oF gateway... ${retry}s" && sleep 5
+
+		NVMEOF_GW_STATUS=$(kubectl_retry -n rook-ceph get cephnvmeofgateway.ceph.rook.io -o jsonpath='{.items[0].status.phase}')
+		[[ "$NVMEOF_GW_STATUS" = "Ready" ]] && break
+	done
+
+	if [ "$retry" -gt "$ROOK_DEPLOY_TIMEOUT" ]; then
+		echo "[Timeout] Ceph NVMe-oF gateway is not running (timeout)"
+		return 1
+	fi
+	echo ""
+}
+
 case "${1:-}" in
 deploy)
 	deploy_rook