NixOS tests

Author: Lillecarl

.github/actions/setup-nix/action.yaml

@@ -19,14 +19,13 @@ runs:
     - name: Install Nix
       uses: nixbuild/nix-quick-install-action@master
       with:
+        # TODO: When ready to migrate from nix-csi to nixkube module naming:
+        # - Update trusted-public-keys: nix-csi.cachix.org-1 → nixkube.cachix.org-1
+        # - Update substituters: https://nix-csi.cachix.org → https://nixkube.cachix.org
+        # - Recreate/migrate Cachix cache from nix-csi to nixkube
         nix_conf: |
-          experimental-features = nix-command flakes
-          access-tokens = github.com=${{ inputs.github_token }}
-          # TODO: When ready to migrate from nix-csi to nixkube module naming:
-          # - Update trusted-public-keys: nix-csi.cachix.org-1 → nixkube.cachix.org-1
-          # - Update substituters: https://nix-csi.cachix.org → https://nixkube.cachix.org
-          # - Recreate/migrate Cachix cache from nix-csi to nixkube
           trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
+          extra-platforms = aarch64-linux qemu-user
           builders = ssh-ng://eu.nixbuild.net aarch64-linux ; ssh-ng://eu.nixbuild.net x86_64-linux
           substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 https://cache.lix.systems?priority=3 ssh-ng://eu.nixbuild.net?priority=4
           keep-env-derivations = true

.github/workflows/ci.yaml

@@ -254,32 +254,6 @@ jobs:
           limit-access-to-actor: true
           wait-timeout-minutes: 5
 
-  test-nixos:
-    needs: build
-    runs-on: ubuntu-latest
-    continue-on-error: true
-    steps:
-      - name: Checkout
-        uses: actions/checkout@main
-
-      - name: Setup Nix environment
-        uses: ./.github/actions/setup-nix
-        with:
-          nixbuild_key: ${{ secrets.NIXBUILD_KEY }}
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          cache_key: test-nixos
-
-      - name: Ensure KVM is available
-        run: |
-          if [ ! -e /dev/kvm ]; then
-            echo "::warning::KVM not available, test will likely fail"
-          fi
-          sudo chmod 666 /dev/kvm || true
-
-      - name: Run NixOS integration test
-        run: |
-          nix build --show-trace --file . nixosTests.containerd --print-build-logs
-
   release:
     needs: [build, test-kind-cache, test-kind-nocache]
     runs-on: ubuntu-latest

.github/workflows/test-nixos.yaml

@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: MIT
+
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+  workflow_dispatch:
+
+env:
+  REPO_USERNAME: ${{ github.actor }}
+  REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+permissions:
+  contents: read
+
+jobs:
+  test-nixos:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@main
+
+      - name: Install Nix
+        uses: nixbuild/nix-quick-install-action@master
+        with:
+          nix_conf: |
+            experimental-features = nix-command flakes
+            access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= nixbuild.net/CHEAKK-1:IvfuR1Ldmkef8pMcSuCl9N2jppEb8KpBzdoB8uKQdxE= nix-csi.cachix.org-1:i4w33gR4efO67jpz8U7g/MdvRQ6mQ3LEF9fB8tES60g=
+            extra-platforms = aarch64-linux
+            builders = ssh-ng://eu.nixbuild.net aarch64-linux ; ssh-ng://eu.nixbuild.net x86_64-linux
+            substituters = https://cache.nixos.org?priority=1 https://nix-csi.cachix.org?priority=2 https://cache.lix.systems?priority=3 ssh-ng://eu.nixbuild.net?priority=4
+
+      - name: Configure SSH Keys
+        shell: bash
+        run: |
+          echo "${{ secrets.NIXBUILD_KEY }}" | install -D -m 600 /dev/stdin ~/.ssh/nixbuild_key
+          install -m 644 "${{ github.workspace }}/.github/assets/ssh-config" ~/.ssh/config
+
+      - name: Setup upterm session
+        uses: owenthereal/action-upterm@v1

default.nix

@@ -169,6 +169,38 @@ rec {
     };
   };
 
+  # Register binfmt handlers for cross-architecture builds (e.g., aarch64-linux on x86_64).
+  #
+  # The kernel binfmt P flag (preserveArgvZero) inserts the original argv[0] as an
+  # extra argument: interpreter <original-argv0> <binary-path> [args...]. A raw
+  # qemu-aarch64 binary doesn't understand this calling convention and will crash.
+  # wrapQemuBinfmtP builds a statically-linked C wrapper (TARGET_QEMU baked in at
+  # compile time) that translates the P-flag calling convention into proper qemu -0
+  # invocation and strips LD_* vars so they can't affect qemu inside Nix sandboxes.
+  #
+  # The kernel registration always points to /run/binfmt/<name> (not the store path
+  # directly) to stay under the kernel's interpreter path length limit and to allow
+  # the symlink to be updated without re-registering.
+  register-binfmt =
+    let
+      qemuBin = lib.getExe' pkgs.qemu-user "qemu-aarch64";
+      wrapper = pkgs.wrapQemuBinfmtP "qemu-aarch64-binfmt-P" qemuBin;
+      wrapperBin = "${wrapper}/bin/qemu-aarch64-binfmt-P";
+    in
+    pkgs.writeShellScriptBin "register-binfmt" ''
+      set -euo pipefail
+      echo "Setting up /run/binfmt/aarch64-linux -> ${wrapperBin}"
+      sudo mkdir -p /run/binfmt
+      sudo ln -sf ${wrapperBin} /run/binfmt/aarch64-linux
+
+      echo "Registering binfmt handler for aarch64-linux..."
+      # Remove any existing registration first (ignore errors if not registered)
+      echo -1 | sudo tee /proc/sys/fs/binfmt_misc/aarch64-linux > /dev/null 2>&1 || true
+      printf ':aarch64-linux:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\x00\xff\xfe\xff\xff:/run/binfmt/aarch64-linux:P\n' | \
+        sudo tee /proc/sys/fs/binfmt_misc/register > /dev/null
+      echo "Done!"
+    '';
+
   lixImage = pkgs.callPackage ./liximage.nix { };
   scratchImage = pkgs.callPackage ./scratchimage.nix { };
 }

tests/nixos/cluster-module.nix

@@ -15,8 +15,11 @@
   environment.systemPackages = [
     pkgs.kubernetes # kubectl, kubeadm, kubelet
     pkgs.cri-tools # crictl
+    pkgs.helix # editor
   ];
 
+  environment.variables.EDITOR = "hx";
+
   # -- Containerd CRI --
   virtualisation.containerd = {
     enable = true;
@@ -130,6 +133,16 @@
     cores = 4;
   };
 
+  # -- Interactive debug user --
+  users.users.nixkube = {
+    isNormalUser = true;
+    initialPassword = "nixkube";
+    extraGroups = [ "wheel" ];
+  };
+
+  # Allow nixkube to use sudo without password
+  security.sudo.wheelNeedsPassword = false;
+
   # Disable COW on etcd data dir (etcd doesn't tolerate COW filesystems)
   system.activationScripts.noCOWs.text = ''
     ${lib.getExe' pkgs.coreutils "mkdir"} --parents /var/lib/etcd
@@ -140,4 +153,11 @@
   systemd.tmpfiles.rules = [
     "d /var/run/nri 0755 root root -"
   ];
+
+  # Enable external networking for image pulls and nix binary cache access
+  networking = {
+    useDHCP = true;
+    firewall.enable = false;
+  };
+  systemd.services.systemd-resolved.enable = true;
 }