work on cache CI

Author: Lillecarl

.github/workflows/ci.yaml

@@ -99,30 +99,52 @@ jobs:
           echo "=== Node pod logs ==="
           kubectl logs -l app.kubernetes.io/name=csi -n nix-csi --tail -1
 
-      - name: Debug on failure
+      - name: Cluster state all
         if: failure()
         run: |
-          echo "=== Cluster state ==="
-          kubectl get all -A
+          echo "=== Cluster state all ==="
+          kubectl get all -A || true
 
-          echo "=== Cache pod logs ==="
-          kubectl logs -l app.kubernetes.io/name=cache -n nix-csi --tail -1 || true
+      - name: Cluster state configmaps
+        if: failure()
+        run: |
+          echo "=== Cluster configmaps ==="
+          kubectl get configmaps -A || true
 
-          echo "=== Cache pod describe ==="
+      - name: Cluster state secrets
+        if: failure()
+        run: |
+          kubectl get secrets -A || true
+
+      - name: Cache
+        if: failure()
+        run: |
+          kubectl logs -l app.kubernetes.io/name=cache -n nix-csi --tail -1 || true
           kubectl describe pod -l app.kubernetes.io/name=cache-n nix-csi || true
 
-          echo "=== Node pod logs ==="
+      - name: Node
+        if: failure()
+        run: |
           kubectl logs -l app.kubernetes.io/name=csi -n nix-csi --tail -1 || true
-
-          echo "=== Node pod describe ==="
           kubectl describe pod -l app.kubernetes.io/name=csi -n nix-csi || true
 
-          echo "=== Init pod logs ==="
+      - name: Init
+        if: failure()
+        run: |
           kubectl logs -l app.kubernetes.io/name=init -n nix-csi --tail -1 || true
 
-          echo "=== Events ==="
+      - name: Events
+        if: failure()
+        run: |
           kubectl get events -n nix-csi --sort-by='.lastTimestamp'
 
+      - name: Troubleshoot with upterm
+        if: failure()
+        uses: owenthereal/action-upterm@main
+        with:
+          limit-access-to-actor: true
+          wait-timeout-minutes: 5
+
   test-kind-nocache:
     needs: build
     runs-on: ubuntu-latest
@@ -184,3 +206,10 @@ jobs:
 
           echo "=== Events ==="
           kubectl get events -n nix-csi --sort-by='.lastTimestamp'
+
+      - name: Troubleshoot with upterm
+        if: failure()
+        uses: owenthereal/action-upterm@main
+        with:
+          limit-access-to-actor: true
+          wait-timeout-minutes: 5

kubenix/secret.nix

@@ -61,30 +61,34 @@ in
         };
       };
       ConfigMap.init-scripts = {
-        data.init-secrets = # bash
+        data.init-secrets = let
+          kc = "kubectl -n ${cfg.namespace}";
+        in
+        # bash
           ''
+            set -x
             mkdir -p /tmp/{ssh-key,nix-key,ssh-auth}
-            if ! kubectl get secret ssh-key &>/dev/null || ! kubectl get configmap ssh-dynauth &>/dev/null; then
+            if ! ${kc} get secret ssh-key &>/dev/null || ! ${kc} get configmap ssh-dynauth &>/dev/null; then
               # Create ssh secret
               ssh-keygen -t ed25519 -C "" -N "" -f /tmp/ssh-key/id_ed25519
-              kubectl delete --ignore-not-found secret ssh-key
-              kubectl create secret generic ssh-key --from-file=/tmp/ssh-key
+              ${kc} delete --ignore-not-found secret ssh-key
+              ${kc} create secret generic ssh-key --from-file=/tmp/ssh-key
 
               # Create pubkey configmap
               cp /tmp/ssh-key/id_ed25519.pub /tmp/ssh-auth/authorized_keys
               echo "* $(cat /tmp/ssh-key/id_ed25519.pub)" > /tmp/ssh-auth/ssh_known_hosts
-              kubectl delete --ignore-not-found configmap ssh-dynauth
-              kubectl create configmap ssh-dynauth --from-file=/tmp/ssh-auth
+              ${kc} delete --ignore-not-found configmap ssh-dynauth
+              ${kc} create configmap ssh-dynauth --from-file=/tmp/ssh-auth
             fi
 
-            if ! kubectl get secret nix-key &>/dev/null; then
+            if ! ${kc} get secret nix-key &>/dev/null; then
               # create nix binary cache key
               nix-store --generate-binary-cache-key \
                 nix-cache-1 \
                 /tmp/nix-key/nix_ed25519 \
                 /tmp/nix-key/nix_ed25519.pub
-              kubectl delete --ignore-not-found secret nix-key
-              kubectl create secret generic nix-key --from-file=/tmp/nix-key
+              ${kc} delete --ignore-not-found secret nix-key
+              ${kc} create secret generic nix-key --from-file=/tmp/nix-key
             fi
           '';
       };

liximage.nix

@@ -42,9 +42,16 @@ rec {
             set -x
             mkdir /tmp
             rsync --archive ${fakeNss}/ /
+
+            # Check if we can SSH to nix-cache
+            EXTRA_SUBSTITUTERS="local?trusted=true"
+            if ssh nix@nix-cache true 2>/dev/null; then
+              EXTRA_SUBSTITUTERS="$EXTRA_SUBSTITUTERS ssh-ng://nix@nix-cache?trusted=true"
+            fi
+
             nix \
               build \
-                --extra-substituters "local?trusted=true ssh-ng://nix@nix-cache?trusted=true" \
+                --extra-substituters "$EXTRA_SUBSTITUTERS" \
                 --max-jobs auto \
                 --option sandbox false \
                 --store /nix-volume \