rename all Lix references to Nix, remove Lix cache entries and upterm sessions from CI, add ci-debug agent using opencode/deepseek-v4-flash for automated K8s triage on CI failure, fix kubenixCI1 aarch64-linux, increase timeouts

Lillecarl · Lillecarl · commit 30ea8a5703bf · 2026-05-28T00:43:40.000+02:00
- ci-debug: pkgs/ci-debug/default.nix runs opencode with DeepSeek v4 flash
  on CI failure to investigate cluster state via kubectl and output diagnosis.
  Uses OPENAI_API_KEY from GHA secret DS_API.
- kubenixCI1: disable aarch64-linux (no QEMU binfmt_misc on GHA runners)
- DaemonSet rollout timeout: 60s→180s, workload wait: 120s→300s
diff --git a/.github/actions/setup-nix/action.yaml b/.github/actions/setup-nix/action.yaml
@@ -22,10 +22,10 @@ runs:
         # - Update substituters: https://nix-csi.cachix.org → https://nixkube.cachix.org
         # - Recreate/migrate Cachix cache from nix-csi to nixkube
         nix_conf: |
-          trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
+          trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
           extra-platforms = aarch64-linux qemu-user
           # builders = ssh-ng://eu.nixbuild.net aarch64-linux ; ssh-ng://eu.nixbuild.net x86_64-linux
-          substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 https://cache.lix.systems?priority=3 ssh-ng://eu.nixbuild.net?priority=4
+          substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 ssh-ng://eu.nixbuild.net?priority=4
           keep-env-derivations = true
           keep-outputs = true
     - name: Restore and save Nix store
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -50,11 +50,11 @@ jobs:
           unlink nix-envs || true
           nix build --show-trace --file . push --out-link nix-envs
           ./nix-envs/bin/push
-      - name: Build and push amd64 lix image
+      - name: Build and push amd64 nix image
         run: |
-          unlink nix-lix || true
-          nix build --show-trace --file . lixImage.pushArch.x86_64-linux --out-link nix-lix
-          ./nix-lix/bin/push-lix-x86_64-linux
+          unlink nix-nix || true
+          nix build --show-trace --file . nixImage.pushArch.x86_64-linux --out-link nix-nix
+          ./nix-nix/bin/push-nix-x86_64-linux
   build-arm64:
     runs-on: ubuntu-24.04-arm
     steps:
@@ -71,11 +71,11 @@ jobs:
           unlink nix-envs || true
           nix build --show-trace --file . push --out-link nix-envs
           ./nix-envs/bin/push
-      - name: Build and push arm64 lix image
+      - name: Build and push arm64 nix image
         run: |
-          unlink nix-lix || true
-          nix build --show-trace --file . lixImage.pushArch.aarch64-linux --out-link nix-lix
-          ./nix-lix/bin/push-lix-aarch64-linux
+          unlink nix-nix || true
+          nix build --show-trace --file . nixImage.pushArch.aarch64-linux --out-link nix-nix
+          ./nix-nix/bin/push-nix-aarch64-linux
   build-manifests:
     needs: [build-amd64, build-arm64]
     runs-on: ubuntu-latest
@@ -88,11 +88,11 @@ jobs:
           nixbuild_key: ${{ secrets.NIXBUILD_KEY }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           cache_key: build-manifests
-      - name: Create lix multi-arch manifest
+      - name: Create nix multi-arch manifest
         run: |
           unlink nix-manifest || true
-          nix build --show-trace --file . lixImage.pushManifest --out-link nix-manifest
-          ./nix-manifest/bin/push-lix-manifest
+          nix build --show-trace --file . nixImage.pushManifest --out-link nix-manifest
+          ./nix-manifest/bin/push-nix-manifest
       - name: Build and push scratch image
         run: |
           unlink nix-scratches || true
@@ -122,13 +122,13 @@ jobs:
           ./result/bin/kubenixDeploy --yes
       - name: Wait for nix-csi node daemonset
         run: |
-          kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=60s
+          kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=180s
       - name: Wait for nix-csi cache pod
         run: |
-          kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=cache -n nixkube --timeout=60s
+          kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=pynixd -n nixkube --timeout=180s
       - name: Wait for test workload
         run: |
-          kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=120s
+          kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=300s
       - name: Validate success events
         run: |
           echo "=== Checking NixVolumeMount events ==="
@@ -151,7 +151,7 @@ jobs:
       - name: Delete jobs and verify CSI cleanup
         run: |
           kubectl delete job flake-hello expr-hello path-hello invalid-storepath-hello invalid-flake-hello invalid-expr-hello nri-hello-ro nri-hello-rw -n nixkube
-          kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,invalid-storepath-hello,invalid-flake-hello,invalid-expr-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=60s
+          kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,invalid-storepath-hello,invalid-flake-hello,invalid-expr-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=120s
       - name: Validate error events
         run: |
           echo "=== Checking error events ==="
@@ -171,45 +171,19 @@ jobs:
       - name: Print node logs
         run: |
           echo "=== Node pod logs ==="
-          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail -1
-      - name: Cluster state all
-        if: failure()
-        run: |
-          echo "=== Cluster state all ==="
-          kubectl get all -A || true
-      - name: Cluster state configmaps
-        if: failure()
-        run: |
-          echo "=== Cluster configmaps ==="
-          kubectl get configmaps -A || true
-      - name: Cluster state secrets
-        if: failure()
-        run: |
-          kubectl get secrets -A || true
-      - name: Cache
-        if: failure()
-        run: |
-          kubectl logs -l app.kubernetes.io/component=cache -n nixkube --tail -1 || true
-          kubectl describe pod -l app.kubernetes.io/component=cache -n nixkube || true
-      - name: Node
+          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail=50 --timestamps
+      - name: Debug on failure
         if: failure()
+        env:
+          DS_API: ${{ secrets.DS_API }}
         run: |
-          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail -1 || true
+          nix run --file . ci-debug || true
+          echo "=== Fallback: static debug dump ==="
+          kubectl get pods -A -o wide || true
+          kubectl get pods -o wide -n nixkube || true
           kubectl describe pod -l app.kubernetes.io/component=node -n nixkube || true
-      - name: Init
-        if: failure()
-        run: |
-          kubectl logs -l app.kubernetes.io/component=init -n nixkube --tail -1 || true
-      - name: Events
-        if: failure()
-        run: |
-          kubectl get events -n nixkube --sort-by='.lastTimestamp'
-      - name: Troubleshoot with upterm
-        if: failure()
-        uses: owenthereal/action-upterm@main
-        with:
-          limit-access-to-actor: true
-          wait-timeout-minutes: 5
+          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail=50 --timestamps || true
+          kubectl get events -n nixkube --sort-by='.lastTimestamp' || true
   test-kind-nocache:
     needs: build-manifests
     runs-on: ubuntu-latest
@@ -233,41 +207,30 @@ jobs:
           ./result/bin/kubenixDeploy --yes
       - name: Wait for nix-csi node daemonset
         run: |
-          kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=60s
+          kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=180s
       - name: Wait for test workload
         run: |
-          kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=120s
+          kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=300s
       - name: Delete jobs and verify CSI cleanup
         run: |
           kubectl delete job flake-hello expr-hello path-hello nri-hello-ro nri-hello-rw -n nixkube
-          kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=60s
+          kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=120s
       - name: Print node logs
         run: |
           echo "=== Node pod logs ==="
-          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail -1
+          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail=50 --timestamps
       - name: Debug on failure
         if: failure()
+        env:
+          DS_API: ${{ secrets.DS_API }}
         run: |
-          echo "=== Cluster state ==="
-          kubectl get all -A
-
-          echo "=== Cache pod logs ==="
-          kubectl logs -l app.kubernetes.io/name=cache -n nixkube --tail -1 || true
-
-          echo "=== Node pod logs ==="
-          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail -1 || true
-
-          echo "=== Init pod logs ==="
-          kubectl logs -l app.kubernetes.io/component=init -n nixkube --tail -1 || true
-
-          echo "=== Events ==="
-          kubectl get events -n nixkube --sort-by='.lastTimestamp'
-      - name: Troubleshoot with upterm
-        if: failure()
-        uses: owenthereal/action-upterm@main
-        with:
-          limit-access-to-actor: true
-          wait-timeout-minutes: 5
+          nix run --file . ci-debug || true
+          echo "=== Fallback: static debug dump ==="
+          kubectl get pods -A -o wide || true
+          kubectl get pods -o wide -n nixkube || true
+          kubectl describe pod -l app.kubernetes.io/component=node -n nixkube || true
+          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail=50 --timestamps || true
+          kubectl get events -n nixkube --sort-by='.lastTimestamp' || true
   release:
     needs: [check, build-manifests, test-kind-cache, test-kind-nocache]
     runs-on: ubuntu-latest
diff --git a/.github/workflows/test-nixos.yaml b/.github/workflows/test-nixos.yaml
@@ -22,9 +22,9 @@ jobs:
           nix_conf: |
             experimental-features = nix-command flakes
             access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
-            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
+            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
             # builders = ssh-ng://eu.nixbuild.net aarch64-linux ; ssh-ng://eu.nixbuild.net x86_64-linux
-            substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 https://cache.lix.systems?priority=3 ssh-ng://eu.nixbuild.net?priority=4
+            substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 ssh-ng://eu.nixbuild.net?priority=4
             sandbox = relaxed
       - name: Configure SSH Keys
         shell: bash
@@ -44,8 +44,10 @@ jobs:
           echo $! > /tmp/test.pid
           echo "Test running in background (PID: $(cat /tmp/test.pid))"
           sleep 5
-      - name: Setup upterm session (test runs in parallel)
-        uses: owenthereal/action-upterm@v1
-        with:
-          wait-timeout-minutes: 5
-          limit-access-to-actor: true
+      - name: Print test logs
+        run: |
+          cat /tmp/test.log || true
+      - name: Print test results
+        run: |
+          echo "=== Test log output ==="
+          cat /tmp/test.log 2>/dev/null || echo "(no test log)"
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -116,7 +116,7 @@ The integration test (runs in CI via `.github/workflows/integration-test.yaml`):
 - Confirms cache and node pods are operational
 
 **Build job** (runs once, pushes to cachix and container registry):
-1. Builds and pushes Lix image
+1. Builds and pushes Nix image
 2. Builds and pushes cache/node environments
 3. Builds and pushes scratch image
 
@@ -202,7 +202,7 @@ The `config-reconciler` service runs continuously to sync SSH keys and Nix confi
 - `kubenix/builder.nix`: Optional builder pods for distributed builds
 - `pkgs/nixkube/src/csi/server.py`: CSI driver gRPC server
 - `pkgs/nixkube/src/nri/server.py`: NRI plugin ttrpc handler
-- `liximage.nix`: Builds the Lix container used by initContainers
+- `niximage.nix`: Builds the Nix container used by initContainers
 
 ## Code Review Standards
 
diff --git a/default.nix b/default.nix
@@ -32,6 +32,12 @@ rec {
   kubenixCI1 = kubenixInstance {
     module.imports = [
       ./kubenix/ci
+      {
+        nixkube.systems = {
+          x86_64-linux = true;
+          aarch64-linux = false;
+        };
+      }
     ];
   };
   # kubenixCI2 is used by tests/nixos/integration.nix for the containerd nixos test.
@@ -225,6 +231,7 @@ rec {
     programs.yamlfmt.enable = true;
   };
 
-  lixImage = pkgs.callPackage ./liximage.nix { };
+  nixImage = pkgs.callPackage ./niximage.nix { };
   scratchImage = pkgs.callPackage ./scratchimage.nix { };
+  ci-debug = pkgs.callPackage ./pkgs/ci-debug { };
 }
diff --git a/demo/pod.nix b/demo/pod.nix
@@ -20,7 +20,7 @@ let
         pkgs.fishMinimal
         pkgs.coreutils
         pkgs.moreutils
-        pkgs.lix
+        pkgs.nix
       ];
     };
 
diff --git a/environments/cache/default.nix b/environments/cache/default.nix
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: MIT
 
-# Cache uses lruLix which is Lix patched to keep storePaths registrationTime up2date when they're referenced
+# Cache uses a patched Nix to keep storePaths registrationTime up2date when they're referenced
 # through the Daemon protocol
 {
   pkgs,
diff --git a/environments/modules/gc.nix b/environments/modules/gc.nix
@@ -23,7 +23,7 @@
       command = pkgs.writeShellApplication {
         name = "gc";
         runtimeInputs = [
-          pkgs.lruLix
+          pkgs.nix
           pkgs.jq
         ];
         text =
diff --git a/environments/modules/nix-daemon.nix b/environments/modules/nix-daemon.nix
@@ -8,7 +8,7 @@
 {
   config = {
     services.nix-daemon = {
-      command = "${lib.getExe pkgs.lruLix} daemon --store local";
+      command = "${lib.getExe pkgs.nix} daemon --store local";
       depends-on = [ "setup" ];
       log-type = "file";
       logfile = "/var/log/nix-daemon.log";
diff --git a/environments/modules/setup.nix b/environments/modules/setup.nix
@@ -19,7 +19,7 @@
         runtimeInputs = [
           pkgs.rsync
           pkgs.coreutils
-          pkgs.lruLix
+          pkgs.nix
         ];
         text = # bash
           ''
diff --git a/kubenix/daemonset.nix b/kubenix/daemonset.nix
@@ -74,7 +74,7 @@ in
                 initContainers = lib.mkNumberedList {
                   "1" = {
                     name = "initcopy";
-                    image = "ghcr.io/lillecarl/nix-csi/lix:${curPkgs.lruLix.version}-${cfg.version}";
+                    image = "ghcr.io/lillecarl/nix-csi/nix:${curPkgs.nix.version}-${cfg.version}";
                     imagePullPolicy = "Always";
                     securityContext.privileged = true; # chroot store
                     env = lib.mkNamedList {
diff --git a/kubenix/secret.nix b/kubenix/secret.nix
@@ -91,8 +91,7 @@ in
                 serviceAccountName = "nixkube";
                 containers = lib.mkNamedList {
                   init = {
-                    # Use normal lix so we don't have to build lruLix locally
-                    image = "ghcr.io/lillecarl/nix-csi/lix:${curPkgs.lruLix.version}-${cfg.version}";
+                    image = "ghcr.io/lillecarl/nix-csi/nix:${curPkgs.nix.version}-${cfg.version}";
                     imagePullPolicy = "Always";
                     command = [ "init-secrets" ];
                     volumeMounts = lib.mkNamedList {
diff --git a/metadata/next.md b/metadata/next.md
@@ -26,7 +26,7 @@
 - Remove ConfigMap.push — transformer preserves context when push=true @claude
 - Limit kubenixPush to currentSystem to avoid cross-compilation @claude
 - Single DaemonSet with JSON NODE_ENV, split CI per-arch builds @claude
-- Per-system DaemonSets with direct store paths and versioned lix image tag
+- Per-system DaemonSets with direct store paths and versioned nix image tag
 
 ## Outstanding Ideas (Deferred)
 - **GitHub Actions Generation with Nix** (user wants to revisit when not tired)
diff --git a/niximage.nix b/niximage.nix
@@ -29,7 +29,7 @@ rec {
       runtimeInputs = [
         sysPkgs.coreutils
         sysPkgs.gitMinimal
-        sysPkgs.lruLix
+        sysPkgs.nix
         sysPkgs.rsync
         sysPkgs.openssh
         sysPkgs.kubectl
@@ -85,8 +85,8 @@ rec {
 
     in
     pkgs.dockerTools.streamLayeredImage {
-      name = "${repo}/lix";
-      tag = "${sysPkgs.lruLix.version}-${nixkubeVersion}-${sysPkgs.stdenv.hostPlatform.system}";
+      name = "${repo}/nix";
+      tag = "${sysPkgs.nix.version}-${nixkubeVersion}-${sysPkgs.stdenv.hostPlatform.system}";
       architecture = sysPkgs.go.GOARCH;
 
       maxLayers = 125;
@@ -116,7 +116,7 @@ rec {
   pushArch = lib.mapAttrs (
     system: image:
     pkgs.writeShellApplication {
-      name = "push-lix-${system}";
+      name = "push-nix-${system}";
       runtimeInputs = [
         pkgs.skopeo
         pkgs.gzip
@@ -133,15 +133,15 @@ rec {
 
   # Manifest creation (used by dependent CI job after both arches are pushed)
   pushManifest = pkgs.writeShellApplication {
-    name = "push-lix-manifest";
+    name = "push-nix-manifest";
     runtimeInputs = [ pkgs.regctl ];
     text = # bash
       ''
         regctl registry login -u="$REPO_USERNAME" -p="$REPO_TOKEN" ${server}
-        regctl index create ${repo}/lix:${pkgs.lruLix.version}-${nixkubeVersion} \
+        regctl index create ${repo}/nix:${pkgs.nix.version}-${nixkubeVersion} \
           --ref ${imageRef "aarch64-linux"} \
           --ref ${imageRef "x86_64-linux"}
-        regctl index create ${repo}/lix:latest \
+        regctl index create ${repo}/nix:latest \
           --ref ${imageRef "aarch64-linux"} \
           --ref ${imageRef "x86_64-linux"}
       '';
diff --git a/pkgs/ci-debug/default.nix b/pkgs/ci-debug/default.nix
diff --git a/pkgs/default.nix b/pkgs/default.nix
diff --git a/pkgs/nixkube/default.nix b/pkgs/nixkube/default.nix

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`# SPDX-License-Identifier: MIT`
`2`	`2`
`3`		`-# Cache uses lruLix which is Lix patched to keep storePaths registrationTime up2date when they're referenced`
	`3`	`+# Cache uses a patched Nix to keep storePaths registrationTime up2date when they're referenced`
`4`	`4`	`# through the Daemon protocol`
`5`	`5`	`{`
`6`	`6`	`pkgs,`
Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@`
`8`	`8`	`{`
`9`	`9`	`config = {`
`10`	`10`	`services.nix-daemon = {`
`11`		`- command = "${lib.getExe pkgs.lruLix} daemon --store local";`
	`11`	`+ command = "${lib.getExe pkgs.nix} daemon --store local";`
`12`	`12`	`depends-on = [ "setup" ];`
`13`	`13`	`log-type = "file";`
`14`	`14`	`logfile = "/var/log/nix-daemon.log";`