feat: Helm-managed node placeholders

.github/workflows/scale-placeholders.yaml

@@ -1,4 +1,4 @@
-name: Scale user placeholders
+name: Scale node placeholders
 on:
   workflow_dispatch:
     inputs:
@@ -42,7 +42,7 @@ jobs:
         go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
     - name: Apply scale-change
       run: |
-        deployer generate replicas ${{ github.event.inputs.cluster }} ${{ github.event.inputs.hub }} ${{ github.event.inputs.replicas }}
+        deployer generate node-replicas ${{ github.event.inputs.cluster }} ${{ github.event.inputs.hub }} ${{ github.event.inputs.replicas }}
     - name: Commit change
       run: |
         # Store branch name
@@ -53,7 +53,7 @@ jobs:
 
         # Add changes in config/clusters of existing tracked files
         git add -u config/clusters
-        git commit -m "Scale placeholders to ${{ github.event.inputs.replicas }} for ${{ github.event.inputs.cluster }}/${{ github.event.inputs.hub }}"
+        git commit -m "Scale node placeholders to ${{ github.event.inputs.replicas }} for ${{ github.event.inputs.cluster }}/${{ github.event.inputs.hub }}"
 
         # Push upstream
         git push --set-upstream origin "${branch_name}"

deployer/dev_commands/generate/placeholders.py

@@ -2,6 +2,8 @@
 Commands for reserving placeholders
 """
 
+from typing import Literal
+
 import typer
 from ruamel.yaml import YAML
 
@@ -12,7 +14,7 @@
 
 
 @generate_app.command()
-def replicas(
+def user_replicas(
     cluster_name: str = typer.Argument(
         "2i2c",
         help="Name of cluster for which to reserve capacity",
@@ -23,6 +25,30 @@ def replicas(
     """
     Scale up user placeholders to reserve capacity via N user placeholders
     """
+    patch_replicas(cluster_name, hub_name, replicas, "user")
+
+
+@generate_app.command()
+def node_replicas(
+    cluster_name: str = typer.Argument(
+        "2i2c",
+        help="Name of cluster for which to reserve capacity",
+    ),
+    hub_name: str = typer.Argument(..., help="Name of hub to operate on"),
+    replicas: int = typer.Argument(...),
+):
+    """
+    Scale up node placeholders to reserve capacity via N node placeholders
+    """
+    patch_replicas(cluster_name, hub_name, replicas, "node")
+
+
+def patch_replicas(
+    cluster_name: str,
+    hub_name: str,
+    replicas: int,
+    kind: Literal["node", "user"],
+):
     cluster = Cluster.from_name(cluster_name)
 
     if replicas < 0:
@@ -46,9 +72,12 @@ def replicas(
     basehub_config = hub_config.get("basehub", hub_config)
 
     try:
-        placeholders = basehub_config["jupyterhub"]["scheduling"]["userPlaceholder"]
+        if kind == "node":
+            placeholders = basehub_config["nodePlaceholder"]
+        else:
+            placeholders = basehub_config["jupyterhub"]["scheduling"]["userPlaceholder"]
     except KeyError as exc:
-        exc.add_note("Could not find userPlaceholders section in hub configuration")
+        exc.add_note(f"Could not find {kind} placeholders section in hub configuration")
         raise
 
     # Scale repliacs

helm-charts/basehub/templates/deployment-node-placeholder.yaml

@@ -0,0 +1,61 @@
+{{- if .Values.nodePlaceholder.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: node-placeholder
+  labels:
+    app: jupyterhub
+    component: node-placeholder
+spec:
+  replicas: {{ .Values.nodePlaceholder.replicas }}
+  selector:
+    matchLabels:
+      app: jupyterhub
+      component: node-placeholder
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 100%
+      maxSurge: 0
+  template:
+    metadata:
+      labels:
+        app: jupyterhub
+        component: node-placeholder
+    spec:
+      {{- with .Values.nodePlaceholder.nodeSelector }}
+      nodeSelector:
+        {{- . | toYaml | nindent 8 }}
+      {{- end }}
+      {{- with concat .Values.nodePlaceholder.tolerations .Values.nodePlaceholder.extraTolerations }}
+      tolerations:
+        {{- . | toYaml | nindent 8 }}
+      {{- end }}
+      affinity:
+        # Require that pods run on user nodes only
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: hub.jupyter.org/node-purpose
+                  operator: In
+                  values: [user]
+        # Require that pods should not run in the same topology
+        podAntiAffinity:
+          # Enforce this at scheduling time
+          requiredDuringSchedulingIgnoredDuringExecution:
+            # Nodes with same host name are considered part of the same topology
+          - topologyKey: kubernetes.io/hostname
+            # Find matching pods to determine multiplicity per topology domain
+            labelSelector:
+              matchLabels:
+                app: jupyterhub
+                component: node-placeholder
+
+      containers:
+      - name: pause
+        image: {{ .Values.nodePlaceholder.pauseContainer.image}}
+        securityContext:
+          runAsUser: 1000
+          allowPrivilegeEscalation: false
+{{- end }}

helm-charts/basehub/values.schema.yaml

@@ -12,6 +12,7 @@ type: object
 additionalProperties: false
 required:
   - nfs
+  - nodePlaceholder
   - inClusterNFS
   - global
   - jupyterhub
@@ -116,6 +117,40 @@ properties:
             description: |
               Optional string to use as the name of the share - defaults to name of the
               hub specified in the cluster.yaml file.
+  nodePlaceholder:
+    type: object
+    additionalProperties: false
+    required:
+      - enabled
+      - replicas
+      - nodeSelector
+      - tolerations
+      - extraTolerations
+      - pauseContainer
+    properties:
+      enabled:
+        type: boolean
+      replicas:
+        type: integer
+        minimum: 0
+      pauseContainer:
+        type: object
+        additionalProperties: false
+        required:
+          - image
+        properties:
+          image:
+            type: string
+      nodeSelector:
+        type: object
+        additionalProperties: true
+        description: |
+          An object with key value pairs representing labels.
+      tolerations: &tolerations-spec
+        type: array
+        description: |
+          Tolerations allow a pod to be scheduled on nodes with taints.
+      extraTolerations: *tolerations-spec
   inClusterNFS:
     type: object
     additionalProperties: false

helm-charts/basehub/values.yaml

@@ -341,6 +341,23 @@ nfs:
       - vers=4.2
     baseShareName: /
 
+nodePlaceholder:
+  enabled: false
+  replicas: 0
+  pauseContainer:
+    image: registry.k8s.io/pause:3.10.1
+  nodeSelector:
+    hub.jupyter.org/node-purpose: user
+  tolerations:
+    # Tolerate tainted jupyterhub user nodes
+    - key: hub.jupyter.org_dedicated
+      value: user
+      effect: NoSchedule
+    - key: hub.jupyter.org/dedicated
+      value: user
+      effect: NoSchedule
+  extraTolerations: []
+
 # Use NFS provided by an in cluster server with the nfs-external-provisioner chart
 inClusterNFS:
   enabled: false
@@ -863,7 +880,7 @@ jupyterhub:
         manage_groups: true
         populate_teams_in_auth_state: true
         scope:
-        - read:org
+          - read:org
       Authenticator:
         # Don't allow test username to login into the hub
         # The test service will still be able to create this hub username
@@ -881,7 +898,7 @@ jupyterhub:
             KubeSpawner.image:
               type: string
               title: User docker image
-              description: Determines languages, libraries and interfaces 
+              description: Determines languages, libraries and interfaces
                 available
               help: Leave this blank to use the default
             Spawner.default_url:
@@ -1041,10 +1058,10 @@ jupyterhub:
     loadRoles:
       jupyterhub-groups-exporter:
         services:
-        - jupyterhub-groups-exporter
+          - jupyterhub-groups-exporter
         scopes:
-        - users
-        - groups
+          - users
+          - groups
     image:
       name: quay.io/2i2c/pilot-hub
       tag: "0.0.1-0.dev.git.15467.h0cadc91ad"