CI work

Author: Lillecarl

default.nix

@@ -42,10 +42,20 @@ rec {
       {
         nixkube.cache.enable = false;
         nixkube.builders.enable = false;
+        # push = true retains Nix string context on DaemonSet store paths so
+        # they become part of the manifest's closure.  The NixOS test VM then
+        # has every path in /nix/store, where nix-serve makes them available
+        # as a substituter for nixkube's separate /var/lib/nix-csi store.
+        nixkube.push = true;
         nixkube.systems = {
           x86_64-linux = true;
           aarch64-linux = false;
         };
+        # 10.113.37.1 is the PTP CNI gateway — the host-side veth IP reachable
+        # from all pods.  nix-serve runs there during the NixOS test.
+        nixkube.node.nixConfig.settings.substituters = [
+          "http://10.113.37.1:5000?trusted=1"
+        ];
       }
     ];
   };

tests/nixos/cluster-module.nix

@@ -16,6 +16,9 @@
     pkgs.kubernetes # kubectl, kubeadm, kubelet
     pkgs.cri-tools # crictl
     pkgs.helix # editor
+    pkgs.k9s
+    pkgs.fish
+    pkgs.stern
   ];
 
   environment.variables.EDITOR = "hx";
@@ -47,20 +50,23 @@
     };
   };
 
-  # -- CNI bridge config --
-  # Simple bridge CNI — no cluster-level setup needed, just this config file.
-  environment.etc."cni/net.d/10-bridge.conflist".text = builtins.toJSON {
+  # -- CNI ptp config --
+  # PTP (point-to-point) CNI: each pod gets a veth pair, no shared bridge.
+  # Traffic between pods is routed at L3 on the host, which lets kube-proxy
+  # iptables/ipvs rules see and rewrite service traffic correctly.
+  environment.etc."cni/net.d/10-ptp.conflist".text = builtins.toJSON {
     cniVersion = "1.0.0";
-    name = "bridge";
+    name = "ptp";
     plugins = [
       {
-        type = "bridge";
-        bridge = "cni0";
-        isGateway = true;
+        type = "ptp";
         ipMasq = true;
         ipam = {
           type = "host-local";
-          ranges = [ [ { subnet = "10.244.0.0/16"; } ] ];
+          subnet = "10.113.37.0/24";
+          # Explicit gateway ensures the host-side veth always gets this IP,
+          # which nixkube pods use to reach nix-serve on the VM.
+          gateway = "10.113.37.1";
           routes = [ { dst = "0.0.0.0/0"; } ];
         };
       }
@@ -137,6 +143,19 @@
     memorySize = 4096;
     diskSize = 10240;
     cores = 4;
+    # cache=unsafe tells QEMU to ack writes immediately without flushing to the
+    # host — eliminates the I/O errors that remount the FS read-only in tests.
+    qemu.drives = lib.mkForce [
+      {
+        name = "root";
+        file = "\"$NIX_DISK_IMAGE\"";
+        driveExtraOpts = {
+          cache = "unsafe";
+          format = "qcow2";
+          "if" = "ide";
+        };
+      }
+    ];
   };
 
   # -- Interactive debug user --
@@ -160,6 +179,16 @@
     "d /var/run/nri 0755 root root -"
   ];
 
+  # Serve the VM's /nix/store as an unsigned binary cache on port 5000.
+  # nixkube pods use this as a trusted substituter so test workload paths can be
+  # fetched without going to the internet (paths pre-populated via additionalPaths).
+  services.nix-serve = {
+    enable = true;
+    package = pkgs.nix-serve-ng;
+    bindAddress = "0.0.0.0";
+    port = 5000;
+  };
+
   # Enable external networking for image pulls and nix binary cache access
   # SLiRP provides DHCP + DNS automatically
   networking.firewall.enable = false;