feat: MU1 multi-user control plane (opt-in, SH_REQUIRE_AUTH=false by default)

Dockerfile

@@ -12,6 +12,7 @@ COPY packages/knative-server/package.json ./packages/knative-server/
 COPY packages/sandbox-relay/package.json ./packages/sandbox-relay/
 COPY packages/work-queue/package.json ./packages/work-queue/
 COPY packages/ibac-stub/package.json ./packages/ibac-stub/
+COPY packages/control-plane/package.json ./packages/control-plane/
 COPY harness/package.json ./harness/
 
 # Copy pi-fork (uses npm, not pnpm — has its own package-lock.json)

Makefile

@@ -1,5 +1,5 @@
 .PHONY: lint fmt test test-deploy typecheck demo-remote-sandbox demo-remote-sandbox-teardown \
-	demo-promoted-workflow demo-promoted-workflow-teardown
+	demo-promoted-workflow demo-promoted-workflow-teardown demo-multiuser demo-multiuser-teardown
 
 lint:
 	pre-commit run --all-files
@@ -43,3 +43,13 @@ demo-promoted-workflow:
 
 demo-promoted-workflow-teardown:
 	bash deploy/knative/demo-promoted-workflow.sh --teardown
+
+# MU1 multi-user demo: two GitHub logins, owned sessions, per-user credentials, and a credential
+# property that holds with the deployment's own key present in the environment. Needs a warm cluster,
+# a GitHub OAuth app with device flow enabled, and two GitHub accounts; it SKIPS with a message
+# otherwise. See docs/specs/2026-09-08-multi-user-control-plane-design.md §10.
+demo-multiuser:
+	bash deploy/knative/demo-multiuser.sh $(DEMO_ARGS)
+
+demo-multiuser-teardown:
+	bash deploy/knative/demo-multiuser.sh --teardown

deploy/knative/SMOKE.md

@@ -102,3 +102,29 @@ pass** — health, scale-to-zero, scale-up-from-zero, 404, plus Redis session re
 response text only, not the harness or the setup.
 
 For storage/SCC, KEDA, Redis and kustomize notes, see [`README-ocp.md`](README-ocp.md).
+
+---
+
+# Multi-user demo (`MULTIUSER_LIVE_SMOKE=1`)
+
+`deploy/knative/demo-multiuser.sh` (`make demo-multiuser`) proves MU1's slice-1 claims against a warm
+cluster: two GitHub device-flow logins, a credential per user, owner-filtered session lists, a 404
+across tenants, `session_mismatch` when a valid token names another session, `token_required` under
+`SH_REQUIRE_AUTH=true`, a `/resources` projection, and — the load-bearing one — `credential_required`
+for a subject with no stored key **while the deployment's own `ANTHROPIC_AUTH_TOKEN` is mounted on the
+Service**.
+
+Gated three ways, and it **skips rather than fails** when any gate is unset:
+
+| Gate                     | Why                                                                                |
+| ------------------------ | ---------------------------------------------------------------------------------- |
+| `MULTIUSER_LIVE_SMOKE=1` | needs a live cluster and two model calls                                           |
+| `SH_GITHUB_CLIENT_ID`    | a GitHub OAuth app with **device flow enabled** (off by default); no client secret |
+| two GitHub accounts      | the subject is attested by GitHub, so the script cannot fabricate two of them      |
+
+It sets `SH_REQUIRE_AUTH=true` for the run — so the property demonstrated is the real one, not the
+permissive default — and restores `false` on exit, including on an aborted run.
+
+**What it does not show:** slice 1's sandbox pool is **shared**. Two users' leaves can be placed on the
+same pod; isolation holds at the API, the session store and the inference credential, not the sandbox.
+The tenant-labelled partition is MU2 (spec §8.2). The script says this out loud.

deploy/knative/control-plane.yaml

@@ -0,0 +1,297 @@
+# MU1 multi-user control plane (docs/specs/2026-09-08-multi-user-control-plane-design.md).
+#
+# OPT-IN. Deliberately NOT in kustomization.yaml: MU1 ships the control plane as an addition, and
+# making it part of the default stack -- alongside flipping SH_REQUIRE_AUTH to true and updating the
+# 14 existing unauthenticated callers -- is MU2 (spec §4.3.1, §10). Apply with:
+#
+#   kubectl apply -f deploy/knative/control-plane.yaml
+#
+# Three Secrets must exist first. None can be defaulted: a literal signing key would let anyone who
+# can read the Deployment mint a token for any subject.
+#
+#   # 1. The Ed25519 signing key. The control plane holds the private half; the data plane gets only
+#   #    the public half, as plain config, so a compromised harness can verify but not mint (spec §5.2).
+#   openssl genpkey -algorithm ed25519 -out /tmp/sh-session.key
+#   kubectl create secret generic sh-session-token-key -n default \
+#     --from-file=SH_SESSION_TOKEN_PRIVATE_KEY=/tmp/sh-session.key
+#   # Publish the matching public key on the harness Service (service.yaml SH_SESSION_TOKEN_PUBLIC_KEYS)
+#   # as `<kid>:<base64 DER SPKI>`, where kid = first 16 hex of sha256(the SPKI DER):
+#   openssl pkey -in /tmp/sh-session.key -pubout -outform DER > /tmp/sh-session.pub.der
+#   printf '%s:%s\n' \
+#     "$(openssl dgst -sha256 -hex /tmp/sh-session.pub.der | awk '{print substr($2,1,16)}')" \
+#     "$(base64 < /tmp/sh-session.pub.der | tr -d '\n')"
+#   shred -u /tmp/sh-session.key
+#
+#   # 2. The KEK, in its OWN Secret so that reading the credential store and reading the key that
+#   #    opens it are two distinct RBAC subjects (spec §6.5).
+#   kubectl create secret generic sh-credential-kek -n default \
+#     --from-literal=SH_CREDENTIAL_KEK="$(openssl rand -base64 32)"
+#
+#   # 3. The shared bearer for the exchange hop, mounted into BOTH tiers from this one Secret
+#   #    (spec §5.3.1). Fail-closed in code: absent => every exchange call is rejected.
+#   kubectl create secret generic sh-exchange-token -n default \
+#     --from-literal=SH_EXCHANGE_TOKEN="$(openssl rand -hex 32)"
+#
+# Then set SH_GITHUB_CLIENT_ID below to a registered GitHub OAuth app with DEVICE FLOW ENABLED -- it is
+# off by default, and it is the likeliest first-run failure (spec §5.1.1). No client secret is needed:
+# the device flow treats the app as a public client.
+#
+# Applying this file on its own does NOT give you a running control plane: the three Secrets above and a
+# non-empty SH_GITHUB_CLIENT_ID are all validated at startup. Fail-at-startup is the intended posture
+# (spec §3.5), but it fails in TWO different ways and only one of them writes a log:
+#
+#   * A MISSING SECRET (sh-session-token-key or sh-credential-kek -- both non-optional refs) is caught by
+#     the kubelet before the container is created, so the pod sits in CreateContainerConfigError and
+#     there is NO container log at all. Use `kubectl describe pod -l app=sh-control-plane -n default`;
+#     the event names the missing Secret.
+#   * An EMPTY SH_GITHUB_CLIENT_ID (or an absent SH_EXCHANGE_TOKEN, whose ref is optional) starts the
+#     container, which then exits on main.ts's `required()` guard -- so CrashLoopBackOff, and
+#     `kubectl logs deploy/sh-control-plane` does name the variable.
+#
+# `kubectl describe pod` covers both; reach for it first.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: sh-credentials
+  labels:
+    # A dedicated namespace is what makes the RBAC containment possible at all: Kubernetes RBAC
+    # filters by resourceNames, never by label, so "read the credential store" can only be a
+    # different permission from "read any Secret in the app namespace" if they are different
+    # namespaces (spec §6.5).
+    app.kubernetes.io/part-of: serverless-harness
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: sh-control-plane
+  namespace: default
+---
+# The SERVING path: get/create/update/patch/delete by exact name. No `list`, no `watch`.
+# The Secret name is derived from the subject (sha256, 16 hex chars), so every access names its
+# object -- which is what makes a list-free Role usable rather than a 403 waiting to happen.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: sh-control-plane-credentials
+  namespace: sh-credentials
+rules:
+  - apiGroups: ['']
+    resources: ['secrets']
+    verbs: ['get', 'create', 'update', 'patch', 'delete']
+---
+# Cleanup of departed users needs `list`, so it lives here and is bound to NOTHING. A Job that needs
+# it binds this Role explicitly; the serving path never gets it (spec §6.5).
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: sh-control-plane-credentials-maintenance
+  namespace: sh-credentials
+rules:
+  - apiGroups: ['']
+    resources: ['secrets']
+    verbs: ['get', 'list', 'delete']
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: sh-control-plane-credentials
+  namespace: sh-credentials
+subjects:
+  - kind: ServiceAccount
+    name: sh-control-plane
+    namespace: default
+roleRef:
+  kind: Role
+  name: sh-control-plane-credentials
+  apiGroup: rbac.authorization.k8s.io
+---
+# /resources reads pod phase (spec §7.4). Pods are not secrets: listing them enumerates no users, so
+# get+list here is not the same kind of grant as it would be on the credential store.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: sh-control-plane-pods
+  namespace: default
+rules:
+  - apiGroups: ['']
+    resources: ['pods']
+    verbs: ['get', 'list']
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: sh-control-plane-pods
+  namespace: default
+subjects:
+  - kind: ServiceAccount
+    name: sh-control-plane
+    namespace: default
+roleRef:
+  kind: Role
+  name: sh-control-plane-pods
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sh-control-plane
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: sh-control-plane
+  template:
+    metadata:
+      labels:
+        app: sh-control-plane
+    spec:
+      serviceAccountName: sh-control-plane
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile: { type: RuntimeDefault }
+      containers:
+        - name: sh-control-plane
+          image: dev.local/serverless-harness:local
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities: { drop: ['ALL'] }
+          # Run from the control-plane package dir, matching the relay precedent
+          # (relay-deployment.yaml): `node --import tsx` resolves tsx relative to the CWD, and the
+          # published image links tsx and redis into each package's own node_modules rather than a
+          # root-hoisted /app/node_modules.
+          workingDir: /app/packages/control-plane
+          command: ['node', '--import', 'tsx', 'src/main.ts']
+          env:
+            - name: HOME
+              value: /tmp
+            - name: SH_CONTROL_PLANE_PORT
+              value: '8080'
+            - name: REDIS_URL
+              value: 'redis://redis.default.svc:6379'
+            - name: SH_CREDENTIAL_NAMESPACE
+              value: sh-credentials
+            - name: SH_SANDBOX_NAMESPACE
+              value: default
+            # Per-deployment and NOT a secret (public client, no client secret). Must be filled in
+            # with a GitHub OAuth app that has device flow enabled (spec §5.1.1).
+            #
+            # THE POD WILL NOT START until this is non-empty: main.ts validates it with the same
+            # `required()` guard as the KEK and the exchange token, and that guard rejects '' as well as
+            # unset, so the container exits with `SH_GITHUB_CLIENT_ID is required` and the Deployment
+            # sits in CrashLoopBackOff. That is deliberate -- a control plane serving device-flow starts
+            # with no client id would 500 every login from a healthy-looking pod -- but it means
+            # `kubectl apply -f control-plane.yaml` ALONE never becomes ready. Set it either by editing
+            # this value before applying, or after applying with:
+            #   kubectl set env deploy/sh-control-plane -n default SH_GITHUB_CLIENT_ID=Iv1.xxxx
+            # (demo-multiuser.sh does the latter, before it waits on the rollout.)
+            - name: SH_GITHUB_CLIENT_ID
+              value: ''
+            # Comma-separated subjects (github:<numeric id>) that get role=admin. Empty by default, so
+            # no deployment has an admin unless one is named.
+            - name: SH_ADMIN_SUBJECTS
+              value: ''
+            # Default false (spec §6.4): a deployment does not silently let one subject spend the
+            # operator's key. When true, SH_OPERATOR_INFERENCE_TOKEN must also be set.
+            - name: SH_ALLOW_OPERATOR_FALLBACK
+              value: 'false'
+            - name: SH_SESSION_TOKEN_PRIVATE_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: sh-session-token-key
+                  key: SH_SESSION_TOKEN_PRIVATE_KEY
+            - name: SH_CREDENTIAL_KEK
+              valueFrom:
+                secretKeyRef:
+                  name: sh-credential-kek
+                  key: SH_CREDENTIAL_KEK
+            - name: SH_EXCHANGE_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: sh-exchange-token
+                  key: SH_EXCHANGE_TOKEN
+                  # optional:true for symmetry with the data-plane side (service.yaml). The exchange is
+                  # fail-closed in code -- an absent value rejects every call rather than opening one --
+                  # so a non-optional ref buys no safety and costs the operator the diagnosis: the
+                  # kubelet refuses to create the container at all, so there is no container log to read.
+                  optional: true
+          ports:
+            - containerPort: 8080
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 8080
+            initialDelaySeconds: 2
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8080
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          resources:
+            requests:
+              memory: '256Mi'
+              cpu: '100m'
+            limits:
+              memory: '512Mi'
+              cpu: '500m'
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: sh-control-plane
+  namespace: default
+spec:
+  selector:
+    app: sh-control-plane
+  ports:
+    - port: 8080
+      targetPort: 8080
+---
+# ADDITIVE egress NetworkPolicy: harness -> control plane on TCP 8080.
+#
+# deploy/knative/harness-egress-policy.yaml is a default-deny egress policy on the harness pod whose
+# allowlist predates MU1: DNS, Redis on 6379, the sandbox relay on 8443, and 0.0.0.0/0 on 443 and 6443.
+# MU1's per-turn credential exchange adds a hop the base allowlist never anticipated --
+# SH_CONTROL_PLANE_URL is `http://sh-control-plane.default.svc:8080` (service.yaml), plaintext h2c on
+# TCP 8080 -- and 8080 matches no rule there. On any cluster that enforces egress (OVN-Kubernetes on
+# OCP, and modern kindnet) the exchange fetch is dropped and every authenticated turn ends in
+# 503 credential_unavailable. Fail-closed, so not a security hole, but MU1's central data path is dead.
+#
+# NetworkPolicies UNION: a pod's allowed egress is the union of every policy selecting it, so a second
+# policy that only ADDS an allow is the correct shape for an opt-in component. harness-egress-policy.yaml
+# belongs to the base stack and is applied by `kubectl apply -k deploy/knative` on every deployment,
+# MU1 or not; it is deliberately left untouched so that opting out of MU1 is exactly "do not apply this
+# file" and the base allowlist never widens for deployments that never run a control plane.
+#
+# The podSelector below must stay identical to harness-egress-policy.yaml's -- Knative stamps
+# `serving.knative.dev/service` on the user pod, and an `app=` selector would match nothing.
+# packages/knative-server/test/control-plane-manifest.test.ts compares the two files and parses the
+# port out of SH_CONTROL_PLANE_URL, so a drift in either fails there rather than in production.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: sh-control-plane-egress-from-harness
+  namespace: default
+spec:
+  podSelector:
+    matchLabels:
+      serving.knative.dev/service: serverless-harness
+  policyTypes:
+    - Egress
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app: sh-control-plane
+      ports:
+        - { protocol: TCP, port: 8080 }