test(bdd): single-cluster PKI feature with a secured LLM invoke

tests/bdd/features/single-cluster-helmfile-pki.feature

@@ -0,0 +1,207 @@
+@ncp-local @single-cluster @helmfile @pki
+Feature: Install a local single-cluster NVCF stack with PKI-secured LLM transport
+  As a self-managed NVCF operator,
+  I want the Helmfile workflow with the LLM PKI addon enabled,
+  so that an LLM function answers invocations over a QUIC tunnel whose
+  trust chain is issued by the stack's own PKI.
+
+  # This feature owns its Helmfile environment (local-bdd-pki) so the
+  # default fixtures and the non-PKI features stay unchanged. PKI
+  # enablement is an install-time value: addons.llm.pki.enabled gates
+  # the nvcf-pki release and the stargate Certificate at render.
+  # The compute plane's trust configuration is written by
+  # tests/bdd/scripts/write-transport-trust-env.sh after the control
+  # plane is installed, because the trust bundle (the OpenBao root CA
+  # public cert plus its nvcf-trust-bundle-v1 fingerprint) only exists
+  # once OpenBao is up. The script replaces the fixture's agentConfig
+  # block, which also drops stargateQUICInsecure: the tunnel runs in
+  # secure mode and NVCA rejects bundle trust combined with insecure
+  # QUIC.
+
+  Rule: Helmfile installs the control plane with the LLM PKI addon
+
+    Background:
+      Given these environment variables are set:
+        | name            |
+        | NGC_API_KEY     |
+        | SAMPLE_NGC_ORG  |
+        | SAMPLE_NGC_TEAM |
+      And I copy the file "tests/bdd/fixtures/self-managed-local-bdd.yaml" to "deploy/stacks/self-managed/environments/local-bdd-pki.yaml"
+      # PKI render contract: dnsNames must cover the router's
+      # advertised hostname (single replica advertises its plain
+      # service DNS name); allowedDomains constrains the OpenBao
+      # signing role; the PKI provisioning hook needs the
+      # nvcf-openbao-migrations tag (same image the openbao chart
+      # runs, whose published default is pinned here).
+      And I update yaml file "deploy/stacks/self-managed/environments/local-bdd-pki.yaml" with keys:
+        | global.imagePullSecrets[0].name               | nvcr-pull-secret                                                   |
+        | global.helm.sources.repository                | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                               |
+        | global.image.repository                       | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                               |
+        | api.env.NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE | nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/stargate-client:0.2.0 |
+        | addons.llm.pki.enabled                        | true                                                               |
+        | addons.llm.pki.dnsNames[0]                    | llm-request-router.nvcf.svc.cluster.local                          |
+        | addons.llm.pki.allowedDomains                 | nvcf.svc.cluster.local                                             |
+        | addons.llm.pki.image.tag                      | 0.16.2                                                             |
+        | observability.profile                         | disabled                                                           |
+      And I copy the file "tests/bdd/fixtures/nvcf-compute-plane-local-bdd.yaml" to "deploy/stacks/nvcf-compute-plane/environments/local-bdd-pki.yaml"
+      And I update yaml file "deploy/stacks/nvcf-compute-plane/environments/local-bdd-pki.yaml" with keys:
+        | global.imagePullSecrets[0].name | nvcr-pull-secret                     |
+        | global.helm.sources.repository  | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
+        | global.image.repository         | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
+        | observability.profile           | disabled                             |
+      And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-bdd-pki-secrets.yaml"
+      And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-bdd-pki-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}"
+      # Conflict precheck: ncp-local-cp's k3d serverlb claims
+      # 0.0.0.0:8080/8443/10081, NATS on 4222, and the worker
+      # callback port 10086, overlapping host ports single-cluster
+      # ncp-local needs. Fail loudly so the operator runs
+      # `make -C tools/ncp-local-cluster destroy-multicluster`
+      # before retrying. `k3d cluster get` exits 1 when absent (k3d v5).
+      Given I run command "k3d cluster get ncp-local-cp"
+      And the command exit code should be 1
+      And a single-cluster ncp-local cluster is running
+      And the "nvcr-pull-secret" image pull secret exists in namespaces:
+        | cassandra-system |
+        | nats-system      |
+        | nvcf             |
+        | api-keys         |
+        | ess              |
+        | sis              |
+        | vault-system     |
+        | nvca-operator    |
+        | cert-manager     |
+
+    @llm-pki-install
+    Scenario: Operator installs the control plane with the PKI addon enabled
+      When I run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-pki"
+
+      Then the command exit code should be 0
+
+      Then these Helm releases should be deployed using context "k3d-ncp-local":
+        | name                      | namespace            |
+        | nats                      | nats-system          |
+        | cert-manager              | cert-manager         |
+        | openbao-server            | vault-system         |
+        | nvcf-pki                  | cert-manager         |
+        | cassandra                 | cassandra-system     |
+        | api-keys                  | api-keys             |
+        | sis                       | sis                  |
+        | api                       | nvcf                 |
+        | nvct-api                  | nvcf                 |
+        | invocation-service        | nvcf                 |
+        | grpc-proxy                | nvcf                 |
+        | ess-api                   | ess                  |
+        | notary-service            | nvcf                 |
+        | admin-issuer-proxy        | api-keys             |
+        | reval                     | nvcf                 |
+        | nats-auth-callout-service | nats-system          |
+        | ingress                   | envoy-gateway-system |
+        | llm-request-router        | nvcf                 |
+        | llm-api-gateway           | nvcf                 |
+
+      # The issuer and the stargate leaf are functional gates for the
+      # secure tunnel: the router cannot serve TLS before cert-manager
+      # writes the stargate-quic-tls Secret.
+      When I run command "kubectl wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m"
+      Then the command exit code should be 0
+
+      When I run command "kubectl wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m"
+      Then the command exit code should be 0
+
+  Rule: The compute plane installs with bundle trust distributed from OpenBao
+
+    Background:
+      Given these environment variables are set:
+        | name      |
+        | NVCF_CLI  |
+        | REPO_ROOT |
+      # This rule depends on the earlier control-plane install scenario
+      # in the same feature run. The @llm-pki-nvca scenario is not a
+      # standalone tag target.
+
+    @llm-pki-nvca
+    Scenario: Operator registers the cluster and installs NVCA with bundle trust
+      When I run command:
+        """
+        make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml
+        """
+      Then the command exit code should be 0
+      And file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-register-values.yaml" should exist
+
+      # Fetch the root CA public cert from OpenBao, compute the
+      # canonical fingerprint, and write the transportTLS bundle block
+      # into the compute environment authored by the Background.
+      When I run command:
+        """
+        tests/bdd/scripts/write-transport-trust-env.sh deploy/stacks/nvcf-compute-plane/environments/local-bdd-pki.yaml
+        """
+      Then the command exit code should be 0
+      And the command output should contain "wrote transportTLS bundle config"
+
+      When I run command:
+        """
+        make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local HELMFILE_ENV=local-bdd-pki NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml
+        """
+      Then the command exit code should be 0
+
+      Then these Helm releases should be deployed using context "k3d-ncp-local":
+        | name          | namespace     |
+        | nvca-operator | nvca-operator |
+
+      When I run command "kubectl rollout status deployment/nvca-operator -n nvca-operator --timeout=10m"
+      Then the command exit code should be 0
+
+      When I run command "kubectl wait nvcfbackend ncp-local -n nvca-operator --for=jsonpath={.status.agentStatus}=healthy --timeout=10m"
+      Then the command exit code should be 0
+
+  Rule: An LLM function answers invocations over the secured tunnel
+
+    # Depends on the earlier install and registration scenarios in this
+    # feature run; not a standalone tag target. The scenario body is
+    # the same as the non-PKI feature's LLM scenario: the invoke
+    # succeeding here proves the trust chain end to end, because the
+    # worker's tunnel to stargate runs in secure mode and validates
+    # the served certificate chain against the injected root.
+    @llm-function-type
+    Scenario: Operator creates, deploys, and invokes an LLM-type function over the secured tunnel
+      When I run command:
+        """
+        ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function create --name bdd-pki-openai-compatible-sample --image nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-openai-compatible-sample:local --function-type LLM --inference-url /v1/chat/completions --inference-port 8000 --health-uri /health --health-port 8000 --health-timeout PT30S --llm-model 'name=openai-compatible-sample,uris=/v1/chat/completions|/v1/embeddings,routingMethod=round_robin'
+        """
+      Then the command exit code should be 0
+
+      When I run command:
+        """
+        ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function deploy create --gpu H100 --instance-type NCP.GPU.H100_8x --backend ncp-local --regions us-west-1 --min-instances 1 --max-instances 1 --timeout 900
+        """
+      Then the command exit code should be 0
+
+      When I run command:
+        """
+        ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml api-key generate --description bdd-pki-openai-compatible-sample --for function --scopes invoke_function,list_functions,queue_details,list_functions_details
+        """
+      Then the command exit code should be 0
+
+      When I run command:
+        """
+        ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke --inference-url /v1/chat/completions --model-name openai-compatible-sample --request-body '{"messages":[{"role":"user","content":"bdd-pki-llm"}]}' --timeout 120
+        """
+      Then the command exit code should be 0
+      And the command output should contain "chat.completion"
+      And the command output should contain "fixed 128-byte response"
+
+      # curl reports only the status code so the assertion cannot
+      # match response-body noise.
+      When I run command:
+        """
+        curl -s -o /dev/null -w "%{http_code}" -X POST http://llm.localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{"model":"unauthenticated/check","messages":[]}'
+        """
+      Then the command exit code should be 0
+      And the command output should contain "401"
+
+      # Leave the GPU capacity free, same as the non-PKI feature.
+      When I run command:
+        """
+        ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function delete --deployment-only
+        """
+      Then the command exit code should be 0

tests/bdd/godog_test.go

@@ -492,6 +492,82 @@ func TestSingleClusterHelmfileFeatureFileWiresToSteps(t *testing.T) {
 	}
 }
 
+// TestSingleClusterHelmfilePKIFeatureFileWiresToSteps runs the PKI
+// Helmfile feature against a fake runner. The feature authors its own
+// local-bdd-pki environment from the same fixtures the non-PKI feature
+// uses and distributes the trust bundle via the helper script, so the
+// wiring test only needs canned results for the script, the LLM
+// invoke, and the no-auth curl.
+func TestSingleClusterHelmfilePKIFeatureFileWiresToSteps(t *testing.T) {
+	t.Setenv("NGC_API_KEY", "test-key")
+	t.Setenv("SAMPLE_NGC_ORG", "test-org")
+	t.Setenv("SAMPLE_NGC_TEAM", "test-team")
+	t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli")
+	t.Setenv("REPO_ROOT", "/repo-root-placeholder")
+	suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{
+		"helm list --all-namespaces --kube-context k3d-ncp-local -o json": {ExitCode: 0, Stdout: helmListAllNamespacesJSON()},
+		"tests/bdd/scripts/write-transport-trust-env.sh deploy/stacks/nvcf-compute-plane/environments/local-bdd-pki.yaml": {
+			ExitCode: 0,
+			Stdout:   "wrote transportTLS bundle config (fingerprint sha256:abc) to deploy/stacks/nvcf-compute-plane/environments/local-bdd-pki.yaml\n",
+		},
+		"/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" +
+			" --inference-url /v1/chat/completions --model-name openai-compatible-sample" +
+			" --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-pki-llm\"}]}' --timeout 120": {
+			ExitCode: 0,
+			Stdout: "Function invocation completed!\n\nResponse:\n" +
+				`{"object":"chat.completion","choices":[{"message":{"content":"This is a fixed 128-byte response from an NVCF-hosted OpenAI-compatible sample, used for load testing and throughput benchmarks."}}]}` +
+				"\n",
+		},
+		`curl -s -o /dev/null -w "%{http_code}" -X POST http://llm.localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{"model":"unauthenticated/check","messages":[]}'`: {
+			ExitCode: 0,
+			Stdout:   "401",
+		},
+		// Conflict precheck: feature asserts the conflicting
+		// multi-cluster control-plane is absent.
+		"k3d cluster get ncp-local-cp": {ExitCode: 1},
+	}))
+	seedHelmfileLocalBDDFixture(t, suite.Config.RepoRoot)
+	seedComputePlaneLocalBDDFixture(t, suite.Config.RepoRoot)
+	seedStackSecretsTemplate(t, suite.Config.RepoRoot)
+	writeHelmfileRegisterValues(t, suite.Config.RepoRoot)
+
+	sc := steps.NewScenarioContext(suite)
+	featurePath := mustResolveFeaturePath(t, "single-cluster-helmfile-pki.feature")
+	var out strings.Builder
+	status := godog.TestSuite{
+		Name: "single-cluster-helmfile-pki-wiring",
+		ScenarioInitializer: func(ctx *godog.ScenarioContext) {
+			steps.RegisterAll(ctx, sc)
+		},
+		Options: &godog.Options{
+			Format: "pretty",
+			Paths:  []string{featurePath},
+			Strict: true,
+			Output: &out,
+		},
+	}.Run()
+	if status != 0 {
+		t.Fatalf("godog suite status = %d\n%s", status, out.String())
+	}
+	if !commandRanThatContains(suite.Runner.(*fakeRunner).runs, "install HELMFILE_ENV=local-bdd-pki") {
+		t.Fatal("PKI helmfile install make target was never invoked")
+	}
+	if !commandRanThatContains(suite.Runner.(*fakeRunner).runs, "kubectl wait clusterissuer nvcf-openbao-pki") {
+		t.Fatal("cluster issuer readiness wait was never invoked")
+	}
+	if !commandRanThatContains(suite.Runner.(*fakeRunner).runs, "write-transport-trust-env.sh") {
+		t.Fatal("trust distribution script was never invoked")
+	}
+	if !commandRanThatContainsAll(suite.Runner.(*fakeRunner).runs,
+		"function create --name bdd-pki-openai-compatible-sample",
+		"--function-type LLM") {
+		t.Fatal("LLM sample function was not created with the LLM function type")
+	}
+	if !commandRanThatContains(suite.Runner.(*fakeRunner).runs, "http://llm.localhost:8080/v1/chat/completions") {
+		t.Fatal("unauthenticated LLM gateway check was never invoked")
+	}
+}
+
 // TestObservabilityControlFeatureFileWiresToSteps runs the live-install
 // observability-control feature against a fake runner. It checks the
 // single-cluster Helmfile path renders and verifies the profile-selected
@@ -1065,6 +1141,7 @@ func helmListAllNamespacesJSON() string {
 {"name":"nats","namespace":"nats-system","status":"deployed"},
 {"name":"cert-manager","namespace":"cert-manager","status":"deployed"},
 {"name":"openbao-server","namespace":"vault-system","status":"deployed"},
+{"name":"nvcf-pki","namespace":"cert-manager","status":"deployed"},
 {"name":"cassandra","namespace":"cassandra-system","status":"deployed"},
 {"name":"api-keys","namespace":"api-keys","status":"deployed"},
 {"name":"sis","namespace":"sis","status":"deployed"},
@@ -1552,6 +1629,15 @@ func TestSingleClusterHelmfile(t *testing.T) {
 	runLiveFeature(t, "single-cluster-helmfile.feature")
 }
 
+// TestSingleClusterHelmfilePKI is the live entry point for the
+// PKI-secured LLM transport Helmfile feature. Skipped under -short.
+func TestSingleClusterHelmfilePKI(t *testing.T) {
+	if testing.Short() {
+		t.Skip("live run skipped under -short")
+	}
+	runLiveFeature(t, "single-cluster-helmfile-pki.feature")
+}
+
 // TestObservabilityControl is the live entry point for the control
 // observability profile feature. Skipped under -short.
 func TestObservabilityControl(t *testing.T) {