ci.yaml

# SPDX-License-Identifier: MIT
on:
  pull_request:
  push:
    branches:
      - '*'
      - '!cidev'
    tags:
      - 'v*'
  workflow_dispatch:
env:
  REPO_USERNAME: ${{ github.actor }}
  REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
permissions:
  contents: write
  packages: write
  actions: write
jobs:
  check:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@main
      - name: Setup Nix environment
        uses: ./.github/actions/setup-nix
        with:
          nixbuild_key: ${{ secrets.NIXBUILD_KEY }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          cache_key: check
      - name: Run type checker
        run: nix-shell --run "pyright pkgs/nixkube/src"
      - name: Check formatting
        run: |
          nix run --file . treefmt
          git diff --exit-code
  build-amd64:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@main
      - name: Setup Nix environment
        uses: ./.github/actions/setup-nix
        with:
          nixbuild_key: ${{ secrets.NIXBUILD_KEY }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          cache_key: build-amd64
      - name: Build and push environments
        run: |
          unlink nix-envs || true
          nix build --show-trace --file . push --out-link nix-envs
          ./nix-envs/bin/push
      - name: Build and push amd64 nix image
        run: |
          unlink nix-nix || true
          nix build --show-trace --file . nixImage.pushArch.x86_64-linux --out-link nix-nix
          ./nix-nix/bin/push-nix-x86_64-linux
  build-arm64:
    runs-on: ubuntu-24.04-arm
    steps:
      - name: Checkout
        uses: actions/checkout@main
      - name: Setup Nix environment
        uses: ./.github/actions/setup-nix
        with:
          nixbuild_key: ${{ secrets.NIXBUILD_KEY }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          cache_key: build-arm64
      - name: Build and push environments
        run: |
          unlink nix-envs || true
          nix build --show-trace --file . push --out-link nix-envs
          ./nix-envs/bin/push
      - name: Build and push arm64 nix image
        run: |
          unlink nix-nix || true
          nix build --show-trace --file . nixImage.pushArch.aarch64-linux --out-link nix-nix
          ./nix-nix/bin/push-nix-aarch64-linux
  build-manifests:
    needs: [build-amd64, build-arm64]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@main
      - name: Setup Nix environment
        uses: ./.github/actions/setup-nix
        with:
          nixbuild_key: ${{ secrets.NIXBUILD_KEY }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          cache_key: build-manifests
      - name: Create nix multi-arch manifest
        run: |
          unlink nix-manifest || true
          nix build --show-trace --file . nixImage.pushManifest --out-link nix-manifest
          ./nix-manifest/bin/push-nix-manifest
      - name: Build and push scratch image
        run: |
          unlink nix-scratches || true
          nix build --show-trace --file . scratchImage.push --out-link nix-scratches
          ./nix-scratches/bin/push
  test-kind-cache:
    needs: build-manifests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@main
      - name: Setup Nix environment
        uses: ./.github/actions/setup-nix
        with:
          nixbuild_key: ${{ secrets.NIXBUILD_KEY }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          cache_key: test-kind-cache
      - name: Create Kind cluster
        uses: helm/kind-action@main
      - name: Clean runner
        run: |
          # Remove cache because permissions can get fucked up preventing kluctl from creating it's cache directory
          sudo rm --recursive --force /home/runner/.cache
      - name: Deploy nix-csi
        run: |
          nix build --show-trace --file . kubenixCI1.deploymentScript
          ./result/bin/kubenixDeploy --yes
      - name: Wait for nix-csi node daemonset
        run: |
          kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=180s
      - name: Wait for nix-csi cache pod
        run: |
          kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=pynixd -n nixkube --timeout=180s
      - name: Wait for test workload
        run: |
          kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=300s
      - name: Validate success events
        run: |
          echo "=== Checking NixVolumeMount events ==="
          EVENTS=$(kubectl get events -n nixkube -o json)

          MOUNT_EVENTS=$(echo "$EVENTS" | jq '[.items[] |
            select(.reason == "NixVolumeMount" and .type == "Normal")] | length')

          echo "Found $MOUNT_EVENTS successful NixVolumeMount events"

          # Show event details: pod name and message with timing
          echo "$EVENTS" | jq -r '.items[] |
            select(.reason == "NixVolumeMount" and .type == "Normal") |
            "  - \(.involvedObject.name): \(.message)"'

          if [ "$MOUNT_EVENTS" -lt 3 ]; then
            echo "ERROR: Expected at least 3 mount events, found $MOUNT_EVENTS"
            exit 1
          fi
      - name: Delete jobs and verify CSI cleanup
        run: |
          kubectl delete job flake-hello expr-hello path-hello invalid-storepath-hello invalid-flake-hello invalid-expr-hello nri-hello-ro nri-hello-rw -n nixkube
          kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,invalid-storepath-hello,invalid-flake-hello,invalid-expr-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=120s
      - name: Validate error events
        run: |
          echo "=== Checking error events ==="
          EVENTS=$(kubectl get events -n nixkube -o json)

          # Check for the three expected error reasons (with Nix prefix)
          for reason in "NixPathBuild" "NixFlakeBuild" "NixExprBuild"; do
            count=$(echo "$EVENTS" | jq --arg r "$reason" '[.items[] |
              select(.reason == $r and .type == "Warning")] | length')
            echo "Found $count '$reason' error event(s)"

            # Show details of the errors
            echo "$EVENTS" | jq --arg r "$reason" -r '.items[] |
              select(.reason == $r and .type == "Warning") |
              "  - \(.involvedObject.name): \(.message)"' || true
          done
      - name: Print node logs
        run: |
          echo "=== Node pod logs ==="
          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail=50 --timestamps
      - name: Debug on failure
        if: failure()
        env:
          DS_API: ${{ secrets.DS_API }}
        run: nix run --file . ci-debug || true
  test-kind-nocache:
    needs: build-manifests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@main
      - name: Setup Nix environment
        uses: ./.github/actions/setup-nix
        with:
          nixbuild_key: ${{ secrets.NIXBUILD_KEY }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          cache_key: test-kind-nocache
      - name: Create Kind cluster
        uses: helm/kind-action@main
      - name: Clean runner
        run: |
          sudo rm --recursive --force /home/runner/.cache
      - name: Deploy nix-csi
        run: |
          nix build --show-trace --file . kubenixCI2.deploymentScript
          ./result/bin/kubenixDeploy --yes
      - name: Wait for nix-csi node daemonset
        run: |
          kubectl rollout status daemonset -l app.kubernetes.io/component=node -n nixkube --timeout=180s
      - name: Wait for test workload
        run: |
          kubectl wait --for=condition=complete job/flake-hello job/expr-hello job/path-hello job/nri-hello-ro job/nri-hello-rw -n nixkube --timeout=300s
      - name: Delete jobs and verify CSI cleanup
        run: |
          kubectl delete job flake-hello expr-hello path-hello nri-hello-ro nri-hello-rw -n nixkube
          kubectl wait --for=delete pod -l "job-name in (flake-hello,expr-hello,path-hello,nri-hello-ro,nri-hello-rw)" -n nixkube --timeout=120s
      - name: Print node logs
        run: |
          echo "=== Node pod logs ==="
          kubectl logs -l app.kubernetes.io/component=node -n nixkube --tail=50 --timestamps
      - name: Debug on failure
        if: failure()
        env:
          DS_API: ${{ secrets.DS_API }}
        run: nix run --file . ci-debug || true
  release:
    needs: [check, build-manifests, test-kind-cache, test-kind-nocache]
    runs-on: ubuntu-latest
    if: startsWith(github.ref, 'refs/tags/v')
    steps:
      - name: Checkout
        uses: actions/checkout@main
      - name: Setup Nix environment
        uses: ./.github/actions/setup-nix
        with:
          nixbuild_key: ${{ secrets.NIXBUILD_KEY }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          cache_key: release
      - name: Render deployment manifest
        run: |
          nix build --show-trace --file . kubenixApply.manifestYAMLFile
          cp result nix-csi-deployment.yaml
      - name: Create GitHub Release
        uses: softprops/action-gh-release@v1
        with:
          files: nix-csi-deployment.yaml
          draft: false
          prerelease: false