leaf-scaledjob.yaml

# deploy/knative/leaf-scaledjob.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledJob
metadata:
  name: leaf-worker
  namespace: default
spec:
  jobTargetRef:
    backoffLimit: 0
    # GC each finished Job (and its pod) 60s after completion via the K8s
    # TTL-after-finished controller — long enough to `kubectl logs` a worker
    # for the Act 2e reveal, short enough that the fleet clears on its own.
    ttlSecondsAfterFinished: 60
    template:
      spec:
        restartPolicy: Never
        serviceAccountName: serverless-harness
        securityContext:
          runAsNonRoot: true
          runAsUser: 65532
          fsGroup: 65532
          seccompProfile: { type: RuntimeDefault }
        containers:
          - name: leaf-job
            image: dev.local/serverless-harness:local
            imagePullPolicy: IfNotPresent
            workingDir: /app/packages/knative-server
            command: ['node', '--import', 'tsx', 'src/leaf-job.ts']
            securityContext:
              allowPrivilegeEscalation: false
              readOnlyRootFilesystem: true
              capabilities: { drop: ['ALL'] }
            # Baseline bounds so a runaway leaf (e.g. large LLM context) can't OOM the node;
            # maxReplicaCount caps pod count, these cap per-pod usage. Tune to observed workloads.
            resources:
              requests: { memory: 256Mi, cpu: 100m }
              limits: { memory: 1Gi, cpu: 500m }
            env:
              - name: HOME
                value: /tmp
              - name: REDIS_URL
                value: 'redis://redis.default.svc:6379'
              - name: SH_MODEL
                value: 'claude-haiku-4-5'
              # Lease from the shared sandbox pool (least-loaded, Redis-backed) instead
              # of pinning one pod — otherwise all maxReplicaCount workers collide on
              # sandbox-0. The harness discovers Running pool pods by this label.
              - name: KAGENTI_SANDBOX_POOL_SELECTOR
                value: 'sh.kagenti.io/sandbox-pool=default'
              - name: KAGENTI_SANDBOX_NAMESPACE
                value: 'default'
              # Soft per-pod lease ADMISSION cap (back-pressure) — NOT a resource limit:
              # up to CAP leaves may hold a lease on one pool pod at once. With P pods this
              # sets target concurrency P*CAP; workers beyond that hit
              # SandboxPoolSaturatedError and requeue. Keep maxReplicaCount aligned to P*CAP
              # (see below). NOTE: to measure a sandbox's true throughput knee, raise CAP
              # high (e.g. 1000) so the lease cap — not the pod — bounds concurrency
              # (e6-saturation.sh does exactly this, on the sync path).
              - name: KAGENTI_SANDBOX_CAP
                value: '3'
              - name: KAGENTI_SANDBOX_LEASE_TTL_MS
                value: '60000'
              - name: LEAF_RESULT_TTL_SECONDS
                value: '86400'
              - name: ANTHROPIC_API_KEY
                valueFrom: { secretKeyRef: { name: llm-credentials, key: api-key } }
              - name: ANTHROPIC_BASE_URL
                valueFrom:
                  { secretKeyRef: { name: llm-credentials, key: base-url, optional: true } }
              - name: ANTHROPIC_AUTH_TOKEN
                valueFrom:
                  { secretKeyRef: { name: llm-credentials, key: auth-token, optional: true } }
            volumeMounts:
              - name: tmp
                mountPath: /tmp
        volumes:
          - name: tmp
            emptyDir: {}
  pollingInterval: 5
  # Cap concurrent worker Jobs. Aligned to the pool's ceiling: P(3 pool pods) *
  # CAP(3 leases/pod) = 9 concurrent leaves. 10 leaves one worker of headroom so
  # a peak burst exercises the SandboxPoolSaturatedError -> requeue back-pressure
  # path (rather than failing). Retune together with KAGENTI_SANDBOX_CAP and the
  # pool size in sandbox-pool.yaml.
  maxReplicaCount: 10
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  # Two redis-streams triggers (KEDA activates the ScaledJob if EITHER is active):
  #  - lagCount: undelivered backlog (entries never XREADGROUP'd) → spawns jobs for NEW work
  #    and is the metric that scales the group to zero when fully drained. (Redis 7+.)
  #  - pendingEntriesCount: the PEL (delivered-but-unacked) → keeps a reclaimer scheduled for a
  #    crashed consumer's stuck entry so XAUTOCLAIM can resume it (gate-7) even with no new work.
  # pendingEntriesCount ALONE deadlocks (PEL stays 0 until something reads; nothing reads until PEL>0).
  # Trade-off: while a leaf runs, its entry sits in the PEL so the pendingEntriesCount trigger spawns extra
  # reclaimer Jobs that find nothing to reclaim (entry heartbeat keeps idleCount < min-idle) and exit "idle",
  # bounded by maxReplicaCount; future optimization could gate reclaim-spawning on min-idle-exceeded entries only.
  triggers:
    - type: redis-streams
      metadata:
        address: redis.default.svc:6379
        stream: leaf-queue
        consumerGroup: leaf-workers
        lagCount: '1'
        activationLagCount: '0'
    - type: redis-streams
      metadata:
        address: redis.default.svc:6379
        stream: leaf-queue
        consumerGroup: leaf-workers
        pendingEntriesCount: '1'