rename all Lix references to Nix, remove Lix cache entries and upterm sessions from CI, add ci-debug agent using opencode+DeepSeek V4 Flash for automated K8s triage on CI failure, fix kubenixCI1 aarch64-linux, consolidate CI substituters

ci-debug uses opencodes agentic loop to let DeepSeek V4 Flash
run kubectl commands interactively in the CI environment,
diagnosing K8s failures with full context.

Consolidated CI substituters into kubenix/ci/default.nix so all
CI variants share the same three substituters: nix-csi.cachix.org,
cache.nixos.org, and nix-serve (10.113.37.1:5000). Nix handles
dead/unreachable substituters gracefully, so having the NixOS-test-only
nix-serve URL in Kind CI is safe. Removed kubenixCI2Kind.

Fixed CI2 substituters overriding shared defaults by moving the
nix-serve URL into the shared CI config alongside the public caches.

Configured via opencode.json with {env:DS_API} variable
substitution so no API keys are stored in the repo.

Author: Lillecarl

.github/actions/setup-nix/action.yaml

@@ -22,10 +22,10 @@ runs:
         # - Update substituters: https://nix-csi.cachix.org → https://nixkube.cachix.org
         # - Recreate/migrate Cachix cache from nix-csi to nixkube
         nix_conf: |
-          trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
+          trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
           extra-platforms = aarch64-linux qemu-user
           # builders = ssh-ng://eu.nixbuild.net aarch64-linux ; ssh-ng://eu.nixbuild.net x86_64-linux
-          substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 https://cache.lix.systems?priority=3 ssh-ng://eu.nixbuild.net?priority=4
+          substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 ssh-ng://eu.nixbuild.net?priority=4
           keep-env-derivations = true
           keep-outputs = true
     - name: Restore and save Nix store

.github/workflows/ci.yaml

@@ -50,11 +50,11 @@ jobs:
           unlink nix-envs || true
           nix build --show-trace --file . push --out-link nix-envs
           ./nix-envs/bin/push
-      - name: Build and push amd64 lix image
+      - name: Build and push amd64 nix image
         run: |
-          unlink nix-lix || true
-          nix build --show-trace --file . lixImage.pushArch.x86_64-linux --out-link nix-lix
-          ./nix-lix/bin/push-lix-x86_64-linux
+          unlink nix-nix || true
+          nix build --show-trace --file . nixImage.pushArch.x86_64-linux --out-link nix-nix
+          ./nix-nix/bin/push-nix-x86_64-linux
   build-arm64:
     runs-on: ubuntu-24.04-arm
     steps:
@@ -71,11 +71,11 @@ jobs:
           unlink nix-envs || true
           nix build --show-trace --file . push --out-link nix-envs
           ./nix-envs/bin/push
-      - name: Build and push arm64 lix image
+      - name: Build and push arm64 nix image
         run: |
-          unlink nix-lix || true
-          nix build --show-trace --file . lixImage.pushArch.aarch64-linux --out-link nix-lix
-          ./nix-lix/bin/push-lix-aarch64-linux
+          unlink nix-nix || true
+          nix build --show-trace --file . nixImage.pushArch.aarch64-linux --out-link nix-nix
+          ./nix-nix/bin/push-nix-aarch64-linux
   build-manifests:
     needs: [build-amd64, build-arm64]
     runs-on: ubuntu-latest
@@ -88,11 +88,11 @@ jobs:
           nixbuild_key: ${{ secrets.NIXBUILD_KEY }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           cache_key: build-manifests
-      - name: Create lix multi-arch manifest
+      - name: Create nix multi-arch manifest
         run: |
           unlink nix-manifest || true
-          nix build --show-trace --file . lixImage.pushManifest --out-link nix-manifest
-          ./nix-manifest/bin/push-lix-manifest
+          nix build --show-trace --file . nixImage.pushManifest --out-link nix-manifest
+          ./nix-manifest/bin/push-nix-manifest
       - name: Build and push scratch image
         run: |
           unlink nix-scratches || true
@@ -122,13 +122,13 @@ jobs:
           ./result/bin/kubenixDeploy --yes
       - name: Wait for nix-csi node daemonset
         run: |
-          kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=60s
+          kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=180s
       - name: Wait for nix-csi cache pod
         run: |
-          kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=cache -n nixkube --timeout=60s
+          kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=pynixd -n nixkube --timeout=180s
       - name: Wait for test workload
         run: |
-          kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=120s
+          kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=300s
       - name: Validate success events
         run: |
           echo "=== Checking NixVolumeMount events ==="
@@ -151,7 +151,7 @@ jobs:
       - name: Delete jobs and verify CSI cleanup
         run: |
           kubectl delete job flake-hello expr-hello path-hello invalid-storepath-hello invalid-flake-hello invalid-expr-hello nri-hello-ro nri-hello-rw -n nixkube
-          kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,invalid-storepath-hello,invalid-flake-hello,invalid-expr-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=60s
+          kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,invalid-storepath-hello,invalid-flake-hello,invalid-expr-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=120s
       - name: Validate error events
         run: |
           echo "=== Checking error events ==="
@@ -171,45 +171,12 @@ jobs:
       - name: Print node logs
         run: |
           echo "=== Node pod logs ==="
-          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail -1
-      - name: Cluster state all
-        if: failure()
-        run: |
-          echo "=== Cluster state all ==="
-          kubectl get all -A || true
-      - name: Cluster state configmaps
-        if: failure()
-        run: |
-          echo "=== Cluster configmaps ==="
-          kubectl get configmaps -A || true
-      - name: Cluster state secrets
-        if: failure()
-        run: |
-          kubectl get secrets -A || true
-      - name: Cache
-        if: failure()
-        run: |
-          kubectl logs -l app.kubernetes.io/component=cache -n nixkube --tail -1 || true
-          kubectl describe pod -l app.kubernetes.io/component=cache -n nixkube || true
-      - name: Node
-        if: failure()
-        run: |
-          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail -1 || true
-          kubectl describe pod -l app.kubernetes.io/component=node -n nixkube || true
-      - name: Init
-        if: failure()
-        run: |
-          kubectl logs -l app.kubernetes.io/component=init -n nixkube --tail -1 || true
-      - name: Events
-        if: failure()
-        run: |
-          kubectl get events -n nixkube --sort-by='.lastTimestamp'
-      - name: Troubleshoot with upterm
+          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail=50 --timestamps
+      - name: Debug on failure
         if: failure()
-        uses: owenthereal/action-upterm@main
-        with:
-          limit-access-to-actor: true
-          wait-timeout-minutes: 5
+        env:
+          DS_API: ${{ secrets.DS_API }}
+        run: nix run --file . ci-debug || true
   test-kind-nocache:
     needs: build-manifests
     runs-on: ubuntu-latest
@@ -233,41 +200,23 @@ jobs:
           ./result/bin/kubenixDeploy --yes
       - name: Wait for nix-csi node daemonset
         run: |
-          kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=60s
+          kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=180s
       - name: Wait for test workload
         run: |
-          kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=120s
+          kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=300s
       - name: Delete jobs and verify CSI cleanup
         run: |
           kubectl delete job flake-hello expr-hello path-hello nri-hello-ro nri-hello-rw -n nixkube
-          kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=60s
+          kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=120s
       - name: Print node logs
         run: |
           echo "=== Node pod logs ==="
-          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail -1
+          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail=50 --timestamps
       - name: Debug on failure
         if: failure()
-        run: |
-          echo "=== Cluster state ==="
-          kubectl get all -A
-
-          echo "=== Cache pod logs ==="
-          kubectl logs -l app.kubernetes.io/name=cache -n nixkube --tail -1 || true
-
-          echo "=== Node pod logs ==="
-          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail -1 || true
-
-          echo "=== Init pod logs ==="
-          kubectl logs -l app.kubernetes.io/component=init -n nixkube --tail -1 || true
-
-          echo "=== Events ==="
-          kubectl get events -n nixkube --sort-by='.lastTimestamp'
-      - name: Troubleshoot with upterm
-        if: failure()
-        uses: owenthereal/action-upterm@main
-        with:
-          limit-access-to-actor: true
-          wait-timeout-minutes: 5
+        env:
+          DS_API: ${{ secrets.DS_API }}
+        run: nix run --file . ci-debug || true
   release:
     needs: [check, build-manifests, test-kind-cache, test-kind-nocache]
     runs-on: ubuntu-latest

.github/workflows/test-nixos.yaml

@@ -22,9 +22,9 @@ jobs:
           nix_conf: |
             experimental-features = nix-command flakes
             access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
-            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
+            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
             # builders = ssh-ng://eu.nixbuild.net aarch64-linux ; ssh-ng://eu.nixbuild.net x86_64-linux
-            substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 https://cache.lix.systems?priority=3 ssh-ng://eu.nixbuild.net?priority=4
+            substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 ssh-ng://eu.nixbuild.net?priority=4
             sandbox = relaxed
       - name: Configure SSH Keys
         shell: bash
@@ -44,8 +44,10 @@ jobs:
           echo $! > /tmp/test.pid
           echo "Test running in background (PID: $(cat /tmp/test.pid))"
           sleep 5
-      - name: Setup upterm session (test runs in parallel)
-        uses: owenthereal/action-upterm@v1
-        with:
-          wait-timeout-minutes: 5
-          limit-access-to-actor: true
+      - name: Print test logs
+        run: |
+          cat /tmp/test.log || true
+      - name: Print test results
+        run: |
+          echo "=== Test log output ==="
+          cat /tmp/test.log 2>/dev/null || echo "(no test log)"

CLAUDE.md

@@ -116,7 +116,7 @@ The integration test (runs in CI via `.github/workflows/integration-test.yaml`):
 - Confirms cache and node pods are operational
 
 **Build job** (runs once, pushes to cachix and container registry):
-1. Builds and pushes Lix image
+1. Builds and pushes Nix image
 2. Builds and pushes cache/node environments
 3. Builds and pushes scratch image
 
@@ -202,7 +202,7 @@ The `config-reconciler` service runs continuously to sync SSH keys and Nix confi
 - `kubenix/builder.nix`: Optional builder pods for distributed builds
 - `pkgs/nixkube/src/csi/server.py`: CSI driver gRPC server
 - `pkgs/nixkube/src/nri/server.py`: NRI plugin ttrpc handler
-- `liximage.nix`: Builds the Lix container used by initContainers
+- `niximage.nix`: Builds the Nix container used by initContainers
 
 ## Code Review Standards
 

default.nix

@@ -21,6 +21,9 @@ in
 rec {
   pkgs = import inputs.nixpkgs {
     inherit system;
+    config = {
+      allowUnfree = true;
+    };
     overlays = [ (import ./pkgs) ];
   };
   inherit inputs;
@@ -32,30 +35,41 @@ rec {
   kubenixCI1 = kubenixInstance {
     module.imports = [
       ./kubenix/ci
+      {
+        nixkube.systems = {
+          x86_64-linux = true;
+          aarch64-linux = false;
+        };
+      }
     ];
   };
   # kubenixCI2 is used by tests/nixos/integration.nix for the containerd nixos test.
   # Disables aarch64-linux to avoid needing cross-compilation support.
   kubenixCI2 = kubenixInstance {
     module.imports = [
       ./kubenix/ci
-      {
-        nixkube.pynixd.enable = false;
-        # push = true retains Nix string context on DaemonSet store paths so
-        # they become part of the manifest's closure.  The NixOS test VM then
-        # has every path in /nix/store, where nix-serve makes them available
-        # as a substituter for nixkube's separate /var/lib/nix-csi store.
-        nixkube.push = true;
-        nixkube.systems = {
-          x86_64-linux = true;
-          aarch64-linux = false;
-        };
-        # 10.113.37.1 is the PTP CNI gateway — the host-side veth IP reachable
-        # from all pods.  nix-serve runs there during the NixOS test.
-        nixkube.node.nixConfig.settings.substituters = [
-          "http://10.113.37.1:5000?trusted=1"
-        ];
-      }
+      (
+        { config, pkgs, ... }:
+        {
+          kluctl.preDeployScript = # bash
+            ''
+              export PATH=${lib.makeBinPath [ pkgs.cachix ]}:$PATH
+              cachix push nix-csi ${config.internal.manifestJSONFile}
+            '';
+          nixkube.pynixd.enable = false;
+          # push = true retains Nix string context on DaemonSet store paths so
+          # they become part of the manifest's closure.  The NixOS test VM then
+          # has every path in /nix/store, where nix-serve makes them available
+          # as a substituter for nixkube's separate /var/lib/nix-csi store.
+          nixkube.push = true;
+          nixkube.systems = {
+            x86_64-linux = true;
+            aarch64-linux = false;
+          };
+          # 10.113.37.1 is the PTP CNI gateway — the host-side veth IP reachable
+          # from all pods.  nix-serve runs there during the NixOS test.
+        }
+      )
     ];
   };
   kubenixLocal = kubenixInstance {
@@ -225,6 +239,7 @@ rec {
     programs.yamlfmt.enable = true;
   };
 
-  lixImage = pkgs.callPackage ./liximage.nix { };
+  nixImage = pkgs.callPackage ./niximage.nix { };
   scratchImage = pkgs.callPackage ./scratchimage.nix { };
+  ci-debug = pkgs.callPackage ./pkgs/ci-debug { };
 }

demo/flake.nix

@@ -6,7 +6,12 @@
   outputs = inputs: {
     packages.x86_64-linux.manifest =
       (inputs.easykubenix.lib.easykubenix {
-        pkgs = import inputs.nixpkgs { system = "x86_64-linux"; };
+        pkgs = import inputs.nixpkgs {
+          system = "x86_64-linux";
+          config = {
+            allowUnfree = true;
+          };
+        };
         modules = [
           (
             { pkgs, lib, ... }:

demo/pod.nix

@@ -8,8 +8,18 @@ let
     "x86_64-linux" = "aarch64-linux";
     "aarch64-linux" = "x86_64-linux";
   };
-  pkgs = import inputs.nixpkgs { inherit system; };
-  pkgsCross = import pkgs.path { system = sysMap.${system}; };
+  pkgs = import inputs.nixpkgs {
+    inherit system;
+    config = {
+      allowUnfree = true;
+    };
+  };
+  pkgsCross = import pkgs.path {
+    system = sysMap.${system};
+    config = {
+      allowUnfree = true;
+    };
+  };
 
   package =
     pkgs:
@@ -20,7 +30,7 @@ let
         pkgs.fishMinimal
         pkgs.coreutils
         pkgs.moreutils
-        pkgs.lix
+        pkgs.nix
       ];
     };
 

environments/cache/default.nix

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: MIT
 
-# Cache uses lruLix which is Lix patched to keep storePaths registrationTime up2date when they're referenced
+# Cache uses a patched Nix to keep storePaths registrationTime up2date when they're referenced
 # through the Daemon protocol
 {
   pkgs,

environments/modules/gc.nix

@@ -23,7 +23,7 @@
       command = pkgs.writeShellApplication {
         name = "gc";
         runtimeInputs = [
-          pkgs.lruLix
+          pkgs.nix
           pkgs.jq
         ];
         text =

environments/modules/nix-daemon.nix

@@ -8,7 +8,7 @@
 {
   config = {
     services.nix-daemon = {
-      command = "${lib.getExe pkgs.lruLix} daemon --store local";
+      command = "${lib.getExe pkgs.nix} daemon --store local";
       depends-on = [ "setup" ];
       log-type = "file";
       logfile = "/var/log/nix-daemon.log";