[CI]: cleanup, breakout 5: separate jobs

.github/workflows/job-test-in-host.yml

@@ -0,0 +1,181 @@
+# This currently test docker and nerdctl on windows (w/o canary)
+# Structure is in to allow testing nerdctl on linux as well, though more work is required to make it functional.
+name: job-test-in-host
+
+on:
+  workflow_call:
+    inputs:
+      timeout:
+        required: true
+        type: number
+      runner:
+        required: true
+        type: string
+      canary:
+        required: false
+        default: false
+        type: boolean
+      binary:
+        required: false
+        default: nerdctl
+        type: string
+      go-version:
+        required: true
+        type: string
+      containerd-version:
+        required: true
+        type: string
+      containerd-sha:
+        required: true
+        type: string
+      containerd-service-sha:
+        required: true
+        type: string
+      windows-cni-version:
+        required: true
+        type: string
+      linux-cni-version:
+        required: true
+        type: string
+      linux-cni-sha:
+        required: true
+        type: string
+
+env:
+  GOTOOLCHAIN: local
+
+jobs:
+  test:
+    name: |
+      ${{ inputs.binary != 'nerdctl' && format('{0} < ', inputs.binary) || '' }}
+      ${{ contains(inputs.runner, 'ubuntu') && ' linux' || ' windows' }}
+      ${{ contains(inputs.runner, 'arm') && '(arm)' || '' }}
+      ${{ contains(inputs.runner, '22.04') && '(old ubuntu)' || '' }}
+      ${{ inputs.canary && ' (canary)' || '' }}
+    timeout-minutes: ${{ inputs.timeout }}
+    runs-on: "${{ inputs.runner }}"
+    defaults:
+      run:
+        shell: bash
+
+    env:
+      SHOULD_RUN: "yes"
+      GO_VERSION: ${{ inputs.go-version }}
+      # Both Docker and nerdctl on linux need rootful right now
+      WITH_SUDO: ${{ contains(inputs.runner, 'ubuntu') }}
+      CONTAINERD_VERSION: ${{ inputs.containerd-version }}
+      CONTAINERD_SHA: ${{ inputs.containerd-sha }}
+
+    steps:
+      - name: "Init: checkout"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          fetch-depth: 1
+
+      - if: ${{ inputs.canary }}
+        name: "Init (canary): retrieve latest go and containerd"
+        run: |
+          latest_go="$(. ./hack/provisioning/version/fetch.sh; go::canary::for::go-setup)"
+          latest_containerd="$(. ./hack/provisioning/version/fetch.sh; github::project::latest "containerd/containerd")"
+
+          [ "$latest_go" == "" ] || \
+            printf "GO_VERSION=%s\n" "$latest_go" >> "$GITHUB_ENV"
+          [ "${latest_containerd:1}" == "$CONTAINERD_VERSION" ] || {
+            printf "CONTAINERD_VERSION=%s\n" "${latest_containerd:1}" >> "$GITHUB_ENV"
+            printf "CONTAINERD_SHA=canary is volatile and I accept the risk\n" >> "$GITHUB_ENV"
+          }
+          if [ "$latest_go" == "" ] && [ "${latest_containerd:1}" == "$CONTAINERD_VERSION" ]; then
+            echo "::warning title=No canary::There is currently no canary versions to test. Steps will not run.";
+            printf "SHOULD_RUN=no\n" >> "$GITHUB_ENV"
+          fi
+
+      - if: ${{ env.SHOULD_RUN == 'yes' }}
+        name: "Init: install go"
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b  # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          check-latest: true
+
+      # XXX RUNNER_OS and generally env is too unreliable
+      # - if: ${{ env.RUNNER_OS == 'Linux' }}
+      - if: ${{ contains(inputs.runner, 'ubuntu') && env.SHOULD_RUN == 'yes' }}
+        name: "Init (linux): prepare host"
+        run: |
+          if [ "${{ contains(inputs.binary, 'docker') }}" == true ]; then
+            echo "::group:: configure cdi for docker"
+            sudo mkdir -p /etc/docker
+            sudo jq '.features.cdi = true' /etc/docker/daemon.json | sudo tee /etc/docker/daemon.json.tmp && sudo mv /etc/docker/daemon.json.tmp /etc/docker/daemon.json
+            sudo systemctl restart docker
+            echo "::endgroup::"
+          else
+            # FIXME: this is missing runc (see top level workflow note about the state of this)
+            echo "::group:: install dependencies"
+            sudo ./hack/provisioning/linux/containerd.sh uninstall
+            ./hack/provisioning/linux/containerd.sh rootful "$CONTAINERD_VERSION" "amd64" "$CONTAINERD_SHA" "${{ inputs.containerd-service-sha }}"
+            sudo ./hack/provisioning/linux/cni.sh uninstall
+            ./hack/provisioning/linux/cni.sh install "${{ inputs.linux-cni-version }}" "amd64" "${{ inputs.linux-cni-sha }}"
+            echo "::endgroup::"
+
+            echo "::group:: build nerctl"
+            go install ./cmd/nerdctl
+            echo "$HOME/go/bin" >> "$GITHUB_PATH"
+            # Since tests are going to run root, we need nerdctl to be in a PATH that will survive `sudo`
+            sudo cp "$(which nerdctl)" /usr/local/bin
+            echo "::endgroup::"
+          fi
+
+          # Register QEMU (tonistiigi/binfmt)
+          # `--install all` will only install emulation for architectures that cannot be natively executed
+          # Since some arm64 platforms do provide native fallback execution for 32 bits,
+          # armv7 emulation may or may not be installed, causing variance in the result of `uname -m`.
+          # To avoid that, we explicitly list the architectures we do want emulation for.
+          docker run --privileged --rm tonistiigi/binfmt --install linux/amd64
+          docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
+          docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
+
+          # FIXME: remove expect when we are done removing unbuffer from tests
+          sudo apt-get install -qq expect
+
+      - if: ${{ contains(inputs.runner, 'windows') && env.SHOULD_RUN == 'yes' }}
+        name: "Init (windows): prepare host"
+        env:
+          ctrdVersion: ${{ env.CONTAINERD_VERSION }}
+        run: |
+          # Install WinCNI
+          echo "::group:: install wincni"
+          GOPATH=$(go env GOPATH) WINCNI_VERSION=${{ inputs.windows-cni-version }} ./hack/provisioning/windows/cni.sh
+          echo "::endgroup::"
+
+          # Install containerd
+          echo "::group:: install containerd"
+          powershell hack/provisioning/windows/containerd.ps1
+          echo "::endgroup::"
+
+          # Install nerdctl
+          echo "::group:: build nerctl"
+          go install ./cmd/nerdctl
+          echo "::endgroup::"
+
+      - if: ${{ env.SHOULD_RUN == 'yes' }}
+        name: "Init: install dev tools"
+        run: |
+          echo "::group:: make install-dev-tools"
+          make install-dev-tools
+          echo "::endgroup::"
+
+      # ipv6 is tested only on linux
+      - if: ${{ contains(inputs.runner, 'ubuntu') && env.SHOULD_RUN == 'yes' }}
+        name: "Run (linux): integration tests (IPv6)"
+        run: |
+          ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-ipv6
+
+      - if: ${{ env.SHOULD_RUN == 'yes' }}
+        name: "Run: integration tests"
+        run: |
+          ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=false
+
+      # FIXME: this must go
+      - if: ${{ env.SHOULD_RUN == 'yes' }}
+        name: "Run: integration tests (flaky)"
+        run: |
+          ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=true

.github/workflows/job-test-in-lima.yml

@@ -0,0 +1,118 @@
+# Currently, Lima job test only for EL, though in the future it could be used to also test FreeBSD or other linux-es
+name: job-test-in-lima
+
+on:
+  workflow_call:
+    inputs:
+      timeout:
+        required: true
+        type: number
+      runner:
+        required: true
+        type: string
+      target:
+        required: true
+        type: string
+      guest:
+        required: true
+        type: string
+
+jobs:
+  test:
+    name: "${{ inputs.guest }} ${{ inputs.target }}"
+    timeout-minutes: ${{ inputs.timeout }}
+    runs-on: "${{ inputs.runner }}"
+    env:
+      TARGET: ${{ inputs.target }}
+    steps:
+      - name: "Init: checkout"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          fetch-depth: 1
+
+      - name: "Init: lima"
+        uses: lima-vm/lima-actions/setup@be564a1408f84557d067b099a475652288074b2e  # v1.0.0
+        id: lima-actions-setup
+
+      - name: "Init: Cache"
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684  # v4.2.3
+        with:
+          path: ~/.cache/lima
+          key: lima-${{ steps.lima-actions-setup.outputs.version }}
+
+      - name: "Init: start the guest VM"
+        run: |
+          set -eux
+          # containerd=none is set because the built-in containerd support conflicts with Docker
+          limactl start \
+            --name=default \
+            --cpus=4 \
+            --memory=12 \
+            --containerd=none \
+            --set '.mounts=null | .portForwards=[{"guestSocket":"/var/run/docker.sock","hostSocket":"{{.Dir}}/sock/docker.sock"}]' \
+            template://${{ inputs.guest }}
+
+      # FIXME: the tests should be directly executed in the VM without nesting Docker inside it
+      # https://github.com/containerd/nerdctl/issues/3858
+      - name: "Init: install dockerd in the guest VM"
+        run: |
+          set -eux
+          lima sudo mkdir -p /etc/systemd/system/docker.socket.d
+          cat <<-EOF | lima sudo tee /etc/systemd/system/docker.socket.d/override.conf
+          [Socket]
+          SocketUser=$(whoami)
+          EOF
+          lima sudo dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
+          lima sudo dnf -q -y install docker-ce --nobest
+          lima sudo systemctl enable --now docker
+
+      - name: "Init: configure the host to use dockerd in the guest VM"
+        run: |
+          set -eux
+          sudo systemctl disable --now docker.service docker.socket
+          export DOCKER_HOST="unix://$(limactl ls --format '{{.Dir}}/sock/docker.sock' default)"
+          echo "DOCKER_HOST=${DOCKER_HOST}" >>$GITHUB_ENV
+          docker info
+          docker version
+
+      - name: "Init: expose GitHub Runtime variables for gha"
+        uses: crazy-max/ghaction-github-runtime@3cb05d89e1f492524af3d41a1c98c83bc3025124  # v3.1.0
+
+      - name: "Init: prepare integration tests"
+        run: |
+          set -eux
+
+          sudo losetup -Dv
+          sudo losetup -lv
+
+          [ "$TARGET" = "rootless" ] && TARGET=test-integration-rootless || TARGET=test-integration
+          docker buildx create --name with-gha --use
+          docker buildx build \
+            --output=type=docker \
+            --cache-from type=gha,scope=test-integration-dependencies-amd64 \
+            -t test-integration --target "${TARGET}" \
+            .
+
+      - name: "Run integration tests"
+        # Presumably, something is broken with the way docker exposes /dev to the container, as it appears to only
+        # randomly work. Mounting /dev does workaround the issue.
+        # This might be due to the old kernel shipped with Alma (4.18), or something else between centos/docker.
+        run: |
+          set -eux
+          if [ "$TARGET" = "rootless" ]; then
+            echo "rootless"
+            docker run -t -v /dev:/dev --rm --privileged test-integration /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=false
+          else
+            echo "rootful"
+            docker run -t -v /dev:/dev --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=false
+          fi
+      - name: "Run: integration tests (flaky)"
+        run: |
+          set -eux
+          if [ "$TARGET" = "rootless" ]; then
+            echo "rootless"
+            docker run -t -v /dev:/dev --rm --privileged test-integration /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=true
+          else
+            echo "rootful"
+            docker run -t -v /dev:/dev --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=true
+          fi

.github/workflows/job-test-in-vagrant.yml

@@ -0,0 +1,60 @@
+# Right now, this is testing solely FreeBSD, but could be used to test other targets.
+# Alternatively, this might get replaced entirely by Lima eventually.
+name: job-test-in-vagrant
+
+on:
+  workflow_call:
+    inputs:
+      timeout:
+        required: true
+        type: number
+      runner:
+        required: true
+        type: string
+
+jobs:
+  test:
+    # Will appear as freebsd / 14 in GitHub UI
+    name: "14"
+    timeout-minutes: ${{ inputs.timeout }}
+    runs-on: "${{ inputs.runner }}"
+    steps:
+      - name: "Init: checkout"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          fetch-depth: 1
+
+      - name: "Init: setup cache"
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684  # v4.2.3
+        with:
+          path: /root/.vagrant.d
+          key: vagrant
+
+      - name: "Init: set up vagrant"
+        run: |
+          # from https://github.com/containerd/containerd/blob/v2.0.2/.github/workflows/ci.yml#L583-L596
+          # which is based on https://github.com/opencontainers/runc/blob/v1.1.8/.cirrus.yml#L41-L49
+          # FIXME: https://github.com/containerd/nerdctl/issues/4163
+          curl -fsSL --proto '=https' --tlsv1.2 https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources
+          sudo apt-get update -qq
+          sudo apt-get install -qq libvirt-daemon libvirt-daemon-system vagrant ovmf
+          # https://github.com/vagrant-libvirt/vagrant-libvirt/issues/1725#issuecomment-1454058646
+          sudo cp /usr/share/OVMF/OVMF_VARS_4M.fd /var/lib/libvirt/qemu/nvram/
+          sudo systemctl enable --now libvirtd
+          sudo apt-get build-dep -qq ruby-libvirt
+          sudo apt-get install -qq --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev
+          # Disable strict dependency enforcement to bypass gem version conflicts during the installation of the vagrant-libvirt plugin.
+          sudo env VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1 vagrant plugin install vagrant-libvirt
+
+      - name: "Init: boot VM"
+        run: |
+          ln -sf Vagrantfile.freebsd Vagrantfile
+          sudo vagrant up --no-tty
+
+      - name: "Run: test-unit"
+        run: sudo vagrant up --provision-with=test-unit
+
+      - name: "Run: test-integration"
+        run: sudo vagrant up --provision-with=test-integration

.github/workflows/job-test-unit.yml

@@ -73,7 +73,7 @@ jobs:
           if [ "$RUNNER_OS" == "Windows" ]; then
             GOPATH=$(go env GOPATH) WINCNI_VERSION=${{ inputs.windows-cni-version }} ./hack/provisioning/windows/cni.sh
           elif [ "$RUNNER_OS" == "Linux" ]; then
-            ./hack/provisioning/linux/cni.sh "${{ inputs.linux-cni-version }}" "amd64" "${{ inputs.linux-cni-sha }}"
+            ./hack/provisioning/linux/cni.sh install "${{ inputs.linux-cni-version }}" "amd64" "${{ inputs.linux-cni-sha }}"
           fi
 
       - if: ${{ env.GO_VERSION != '' }}