Skip to content

Commit 19c22a0

Browse files
LillecarlLillecarl
committed
rename all Lix references to Nix, remove Lix cache entries and upterm sessions from CI, add ci-debug agent using curl+DeepSeek API for automated K8s triage on CI failure, fix kubenixCI1 aarch64-linux, increase timeouts
ci-debug gathers cluster state (pods, describe, logs, events, CSI info) and sends it to DeepSeek v4 flash via direct API call for diagnosis. Replaced opencode approach which had API format compatibility issues.
1 parent 26632d2 commit 19c22a0

17 files changed

Lines changed: 196 additions & 119 deletions

File tree

‎.github/actions/setup-nix/action.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,10 @@ runs:
2222
# - Update substituters: https://nix-csi.cachix.org → https://nixkube.cachix.org
2323
# - Recreate/migrate Cachix cache from nix-csi to nixkube
2424
nix_conf: |
25-
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
25+
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
2626
extra-platforms = aarch64-linux qemu-user
2727
# builders = ssh-ng://eu.nixbuild.net aarch64-linux ; ssh-ng://eu.nixbuild.net x86_64-linux
28-
substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 https://cache.lix.systems?priority=3 ssh-ng://eu.nixbuild.net?priority=4
28+
substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 ssh-ng://eu.nixbuild.net?priority=4
2929
keep-env-derivations = true
3030
keep-outputs = true
3131
- name: Restore and save Nix store

‎.github/workflows/ci.yaml‎

Lines changed: 38 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,11 @@ jobs:
5050
unlink nix-envs || true
5151
nix build --show-trace --file . push --out-link nix-envs
5252
./nix-envs/bin/push
53-
- name: Build and push amd64 lix image
53+
- name: Build and push amd64 nix image
5454
run: |
55-
unlink nix-lix || true
56-
nix build --show-trace --file . lixImage.pushArch.x86_64-linux --out-link nix-lix
57-
./nix-lix/bin/push-lix-x86_64-linux
55+
unlink nix-nix || true
56+
nix build --show-trace --file . nixImage.pushArch.x86_64-linux --out-link nix-nix
57+
./nix-nix/bin/push-nix-x86_64-linux
5858
build-arm64:
5959
runs-on: ubuntu-24.04-arm
6060
steps:
@@ -71,11 +71,11 @@ jobs:
7171
unlink nix-envs || true
7272
nix build --show-trace --file . push --out-link nix-envs
7373
./nix-envs/bin/push
74-
- name: Build and push arm64 lix image
74+
- name: Build and push arm64 nix image
7575
run: |
76-
unlink nix-lix || true
77-
nix build --show-trace --file . lixImage.pushArch.aarch64-linux --out-link nix-lix
78-
./nix-lix/bin/push-lix-aarch64-linux
76+
unlink nix-nix || true
77+
nix build --show-trace --file . nixImage.pushArch.aarch64-linux --out-link nix-nix
78+
./nix-nix/bin/push-nix-aarch64-linux
7979
build-manifests:
8080
needs: [build-amd64, build-arm64]
8181
runs-on: ubuntu-latest
@@ -88,11 +88,11 @@ jobs:
8888
nixbuild_key: ${{ secrets.NIXBUILD_KEY }}
8989
github_token: ${{ secrets.GITHUB_TOKEN }}
9090
cache_key: build-manifests
91-
- name: Create lix multi-arch manifest
91+
- name: Create nix multi-arch manifest
9292
run: |
9393
unlink nix-manifest || true
94-
nix build --show-trace --file . lixImage.pushManifest --out-link nix-manifest
95-
./nix-manifest/bin/push-lix-manifest
94+
nix build --show-trace --file . nixImage.pushManifest --out-link nix-manifest
95+
./nix-manifest/bin/push-nix-manifest
9696
- name: Build and push scratch image
9797
run: |
9898
unlink nix-scratches || true
@@ -122,13 +122,13 @@ jobs:
122122
./result/bin/kubenixDeploy --yes
123123
- name: Wait for nix-csi node daemonset
124124
run: |
125-
kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=60s
125+
kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=180s
126126
- name: Wait for nix-csi cache pod
127127
run: |
128-
kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=cache -n nixkube --timeout=60s
128+
kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=pynixd -n nixkube --timeout=180s
129129
- name: Wait for test workload
130130
run: |
131-
kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=120s
131+
kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=300s
132132
- name: Validate success events
133133
run: |
134134
echo "=== Checking NixVolumeMount events ==="
@@ -151,7 +151,7 @@ jobs:
151151
- name: Delete jobs and verify CSI cleanup
152152
run: |
153153
kubectl delete job flake-hello expr-hello path-hello invalid-storepath-hello invalid-flake-hello invalid-expr-hello nri-hello-ro nri-hello-rw -n nixkube
154-
kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,invalid-storepath-hello,invalid-flake-hello,invalid-expr-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=60s
154+
kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,invalid-storepath-hello,invalid-flake-hello,invalid-expr-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=120s
155155
- name: Validate error events
156156
run: |
157157
echo "=== Checking error events ==="
@@ -171,45 +171,19 @@ jobs:
171171
- name: Print node logs
172172
run: |
173173
echo "=== Node pod logs ==="
174-
kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail -1
175-
- name: Cluster state all
176-
if: failure()
177-
run: |
178-
echo "=== Cluster state all ==="
179-
kubectl get all -A || true
180-
- name: Cluster state configmaps
181-
if: failure()
182-
run: |
183-
echo "=== Cluster configmaps ==="
184-
kubectl get configmaps -A || true
185-
- name: Cluster state secrets
186-
if: failure()
187-
run: |
188-
kubectl get secrets -A || true
189-
- name: Cache
190-
if: failure()
191-
run: |
192-
kubectl logs -l app.kubernetes.io/component=cache -n nixkube --tail -1 || true
193-
kubectl describe pod -l app.kubernetes.io/component=cache -n nixkube || true
194-
- name: Node
174+
kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail=50 --timestamps
175+
- name: Debug on failure
195176
if: failure()
177+
env:
178+
DS_API: ${{ secrets.DS_API }}
196179
run: |
197-
kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail -1 || true
180+
nix run --file . ci-debug || true
181+
echo "=== Fallback: static debug dump ==="
182+
kubectl get pods -A -o wide || true
183+
kubectl get pods -o wide -n nixkube || true
198184
kubectl describe pod -l app.kubernetes.io/component=node -n nixkube || true
199-
- name: Init
200-
if: failure()
201-
run: |
202-
kubectl logs -l app.kubernetes.io/component=init -n nixkube --tail -1 || true
203-
- name: Events
204-
if: failure()
205-
run: |
206-
kubectl get events -n nixkube --sort-by='.lastTimestamp'
207-
- name: Troubleshoot with upterm
208-
if: failure()
209-
uses: owenthereal/action-upterm@main
210-
with:
211-
limit-access-to-actor: true
212-
wait-timeout-minutes: 5
185+
kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail=50 --timestamps || true
186+
kubectl get events -n nixkube --sort-by='.lastTimestamp' || true
213187
test-kind-nocache:
214188
needs: build-manifests
215189
runs-on: ubuntu-latest
@@ -233,41 +207,30 @@ jobs:
233207
./result/bin/kubenixDeploy --yes
234208
- name: Wait for nix-csi node daemonset
235209
run: |
236-
kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=60s
210+
kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=180s
237211
- name: Wait for test workload
238212
run: |
239-
kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=120s
213+
kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=300s
240214
- name: Delete jobs and verify CSI cleanup
241215
run: |
242216
kubectl delete job flake-hello expr-hello path-hello nri-hello-ro nri-hello-rw -n nixkube
243-
kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=60s
217+
kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=120s
244218
- name: Print node logs
245219
run: |
246220
echo "=== Node pod logs ==="
247-
kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail -1
221+
kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail=50 --timestamps
248222
- name: Debug on failure
249223
if: failure()
224+
env:
225+
DS_API: ${{ secrets.DS_API }}
250226
run: |
251-
echo "=== Cluster state ==="
252-
kubectl get all -A
253-
254-
echo "=== Cache pod logs ==="
255-
kubectl logs -l app.kubernetes.io/name=cache -n nixkube --tail -1 || true
256-
257-
echo "=== Node pod logs ==="
258-
kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail -1 || true
259-
260-
echo "=== Init pod logs ==="
261-
kubectl logs -l app.kubernetes.io/component=init -n nixkube --tail -1 || true
262-
263-
echo "=== Events ==="
264-
kubectl get events -n nixkube --sort-by='.lastTimestamp'
265-
- name: Troubleshoot with upterm
266-
if: failure()
267-
uses: owenthereal/action-upterm@main
268-
with:
269-
limit-access-to-actor: true
270-
wait-timeout-minutes: 5
227+
nix run --file . ci-debug || true
228+
echo "=== Fallback: static debug dump ==="
229+
kubectl get pods -A -o wide || true
230+
kubectl get pods -o wide -n nixkube || true
231+
kubectl describe pod -l app.kubernetes.io/component=node -n nixkube || true
232+
kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail=50 --timestamps || true
233+
kubectl get events -n nixkube --sort-by='.lastTimestamp' || true
271234
release:
272235
needs: [check, build-manifests, test-kind-cache, test-kind-nocache]
273236
runs-on: ubuntu-latest

‎.github/workflows/test-nixos.yaml‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ jobs:
2222
nix_conf: |
2323
experimental-features = nix-command flakes
2424
access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
25-
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
25+
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
2626
# builders = ssh-ng://eu.nixbuild.net aarch64-linux ; ssh-ng://eu.nixbuild.net x86_64-linux
27-
substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 https://cache.lix.systems?priority=3 ssh-ng://eu.nixbuild.net?priority=4
27+
substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 ssh-ng://eu.nixbuild.net?priority=4
2828
sandbox = relaxed
2929
- name: Configure SSH Keys
3030
shell: bash
@@ -44,8 +44,10 @@ jobs:
4444
echo $! > /tmp/test.pid
4545
echo "Test running in background (PID: $(cat /tmp/test.pid))"
4646
sleep 5
47-
- name: Setup upterm session (test runs in parallel)
48-
uses: owenthereal/action-upterm@v1
49-
with:
50-
wait-timeout-minutes: 5
51-
limit-access-to-actor: true
47+
- name: Print test logs
48+
run: |
49+
cat /tmp/test.log || true
50+
- name: Print test results
51+
run: |
52+
echo "=== Test log output ==="
53+
cat /tmp/test.log 2>/dev/null || echo "(no test log)"

‎CLAUDE.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ The integration test (runs in CI via `.github/workflows/integration-test.yaml`):
116116
- Confirms cache and node pods are operational
117117

118118
**Build job** (runs once, pushes to cachix and container registry):
119-
1. Builds and pushes Lix image
119+
1. Builds and pushes Nix image
120120
2. Builds and pushes cache/node environments
121121
3. Builds and pushes scratch image
122122

@@ -202,7 +202,7 @@ The `config-reconciler` service runs continuously to sync SSH keys and Nix confi
202202
- `kubenix/builder.nix`: Optional builder pods for distributed builds
203203
- `pkgs/nixkube/src/csi/server.py`: CSI driver gRPC server
204204
- `pkgs/nixkube/src/nri/server.py`: NRI plugin ttrpc handler
205-
- `liximage.nix`: Builds the Lix container used by initContainers
205+
- `niximage.nix`: Builds the Nix container used by initContainers
206206

207207
## Code Review Standards
208208

‎default.nix‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ rec {
3232
kubenixCI1 = kubenixInstance {
3333
module.imports = [
3434
./kubenix/ci
35+
{
36+
nixkube.systems = {
37+
x86_64-linux = true;
38+
aarch64-linux = false;
39+
};
40+
}
3541
];
3642
};
3743
# kubenixCI2 is used by tests/nixos/integration.nix for the containerd nixos test.
@@ -225,6 +231,7 @@ rec {
225231
programs.yamlfmt.enable = true;
226232
};
227233

228-
lixImage = pkgs.callPackage ./liximage.nix { };
234+
nixImage = pkgs.callPackage ./niximage.nix { };
229235
scratchImage = pkgs.callPackage ./scratchimage.nix { };
236+
ci-debug = pkgs.callPackage ./pkgs/ci-debug { };
230237
}

‎demo/pod.nix‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ let
2020
pkgs.fishMinimal
2121
pkgs.coreutils
2222
pkgs.moreutils
23-
pkgs.lix
23+
pkgs.nix
2424
];
2525
};
2626

‎environments/cache/default.nix‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# SPDX-License-Identifier: MIT
22

3-
# Cache uses lruLix which is Lix patched to keep storePaths registrationTime up2date when they're referenced
3+
# Cache uses a patched Nix to keep storePaths registrationTime up2date when they're referenced
44
# through the Daemon protocol
55
{
66
pkgs,

‎environments/modules/gc.nix‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
command = pkgs.writeShellApplication {
2424
name = "gc";
2525
runtimeInputs = [
26-
pkgs.lruLix
26+
pkgs.nix
2727
pkgs.jq
2828
];
2929
text =

‎environments/modules/nix-daemon.nix‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
{
99
config = {
1010
services.nix-daemon = {
11-
command = "${lib.getExe pkgs.lruLix} daemon --store local";
11+
command = "${lib.getExe pkgs.nix} daemon --store local";
1212
depends-on = [ "setup" ];
1313
log-type = "file";
1414
logfile = "/var/log/nix-daemon.log";

‎environments/modules/setup.nix‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
runtimeInputs = [
2020
pkgs.rsync
2121
pkgs.coreutils
22-
pkgs.lruLix
22+
pkgs.nix
2323
];
2424
text = # bash
2525
''

0 commit comments

Comments
 (0)