Migrate the outer DOJO appliance to native NixOS services

.claude/skills/run-tests/SKILL.md

@@ -21,22 +21,13 @@ tail -f /tmp/test.log
 
 ## Running Specific Tests
 
-To run only specific test files quickly, edit `deploy.sh` line:
-```bash
-test_container pytest --order-dependencies --timeout=60 -v . "$@"
-```
-
-Change it to run a specific file:
-```bash
-test_container pytest --order-dependencies --timeout=60 -v test_belts.py "$@"
-```
+Pass a pytest expression through the deployment helper:
 
-Then run:
 ```bash
-./deploy.sh -b -t 2>&1 | tee /tmp/test.log &
+./deploy.sh -N -t -k test_create_dojo 2>&1 | tee /tmp/test.log &
 ```
 
-**Important:** After specific tests pass, restore the original line and rerun ALL tests.
+Start the DOJO with `./deploy.sh -b` first if an instance is not already running.
 
 ## Analyzing Failures
 
@@ -50,14 +41,14 @@ After tests complete, analyze `/tmp/test.log`:
 1. **Identify failures** - Analyze `/tmp/test.log` for FAILED/ERROR
 2. **Find root cause** - Look at tracebacks, check CTFd logs with:
    ```bash
-   docker exec $(basename "$PWD") docker logs ctfd 2>&1 | tail -100
+   docker exec "$(basename "$PWD")" journalctl -b -u dojo-ctfd --no-pager
    ```
-   You can also look at all any of the containers inside of `docker exec $(basename "$PWD")`, the whole list including names is in docker-compose.yml.
+   Inspect all native services with `systemctl status dojo.target` and `systemctl --failed` inside the outer container.
 3. **If root cause unclear** - Add logging to help understand, then rerun tests
 4. **Fix the root cause** - Don't just fix symptoms
-5. **Run specific tests** - Edit deploy.sh to run just the affected test file
+5. **Run specific tests** - Pass a `-k` expression to `./deploy.sh -N -t`
 6. **Verify fix** - Ensure the specific test passes
-7. **Run ALL tests** - Restore deploy.sh and run full suite before declaring success
+7. **Run ALL tests** - Run the full suite before declaring success
 
 ## Key Commands
 
@@ -72,13 +63,13 @@ jobs
 cat /tmp/test.log
 
 # Search for failures
-grep -E "(FAILED|ERROR|error)" /tmp/test.log
+rg "FAILED|ERROR|error" /tmp/test.log
 
 # View CTFd logs for debugging
-docker exec $(basename "$PWD") docker logs ctfd 2>&1 | tail -200
+docker exec "$(basename "$PWD")" journalctl -b -u dojo-ctfd --no-pager
 
 # Run DB queries for debugging
-docker exec -i $(basename "$PWD") dojo db
+docker exec -i "$(basename "$PWD")" dojo db
 ```
 
 ## Multinode Testing

.devcontainer/devcontainer.json

@@ -1,12 +1,10 @@
 {
     "name": "Development",
-    "build": {
-        "dockerfile": "../Dockerfile",
-        "context": ".."
-    },
+    "initializeCommand": "cd \"${localWorkspaceFolder}\" && ./nix/build-image.sh pwncollege/dojo-devcontainer:local",
+    "image": "pwncollege/dojo-devcontainer:local",
     "workspaceFolder": "/opt/pwn.college",
     "workspaceMount": "source=${localWorkspaceFolder},target=/opt/pwn.college,type=bind",
-    "runArgs": ["-v", "/tmp/dojo-data-${devcontainerId}:/data"],
+    "runArgs": ["-v", "/tmp/dojo-data-${devcontainerId}:/data:shared"],
     "privileged": true,
     "overrideCommand": false,
     "forwardPorts": [22, 80, 443],

.github/workflows/ci.yml

@@ -11,9 +11,10 @@ jobs:
   test:
     name: "${{ matrix.mode }} ${{ matrix.tier }} Tests"
     runs-on: ubuntu-latest
-    timeout-minutes: 90
+    timeout-minutes: 150
     env:
       DISCORD_CLIENT_SECRET: test-discord-client-secret
+      CI_DATA_ROOT: /mnt/data/dojo-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.mode }}-${{ matrix.tier }}
     strategy:
       matrix:
         include:
@@ -35,7 +36,8 @@ jobs:
             coverage: false
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
+      - uses: cachix/install-nix-action@v31
 
       - name: Host information
         run: |
@@ -63,17 +65,8 @@ jobs:
           docker images
           echo "::endgroup::"
 
-      - name: Set cache mode
-        run: |
-          if [ "${{ matrix.mode }}" = "singlenode" ] && [ "${{ matrix.tier }}" = "semantic" ] && \
-             { [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; }; then
-            echo "SAVE_CACHE=yes" >> "$GITHUB_ENV"
-          else
-            echo "LOAD_CACHE=yes" >> "$GITHUB_ENV"
-            sudo chown $USER:$USER /mnt
-          fi
-
       - name: Free runner disk space
+        if: runner.environment == 'github-hosted'
         run: |
           # The suite fills the runner's root filesystem, which shows up as
           # container starts failing rather than as an obvious disk error.
@@ -82,53 +75,22 @@ jobs:
           echo "::endgroup::"
           sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
                       /usr/local/share/boost /usr/local/share/powershell \
-                      /usr/share/swift /opt/hostedtoolcache "$AGENT_TOOLSDIRECTORY" || true
+                      /usr/share/swift /opt/hostedtoolcache || true
           echo "::group::After"
           df -h /
           echo "::endgroup::"
 
-      - name: Restore docker and workspace cache
-        if: env.LOAD_CACHE == 'yes'
-        uses: actions/cache/restore@v4
-        with:
-          path: |
-            /mnt/cache.img
-          key: dojo-cache-${{ matrix.mode }}-${{ github.run_id }}
-          restore-keys: |
-            dojo-cache-
-
-      - name: Setup cache filesystem
+      - name: Set up test storage
         run: |
-          sudo chown root:root /mnt
-          set -x
-          df -h
-          sudo mkdir -p /mnt/cache
-          sudo mkdir -p /mnt/data
-          if [ ! -e /mnt/cache.img ]; then
-            sudo fallocate -l 9.8G /mnt/cache.img
-            sudo mkfs.btrfs -L cache /mnt/cache.img
-          fi
-
-          if [ "$SAVE_CACHE" != "yes" ]; then
-            sudo truncate -s +20G /mnt/cache.img
-          fi
-
-          sudo mount -o loop,compress=zstd:9 /mnt/cache.img /mnt/cache
-          sudo btrfs filesystem resize max /mnt/cache
-
-          sudo chown $USER:$USER /mnt/cache
-          sudo chown $USER:$USER /mnt/data
-          mkdir -p /mnt/cache/dojo-image
-          mkdir -p /mnt/cache/test-image
-          mkdir -p /mnt/data/docker
-          mkdir -p /mnt/data/workspace
+          sudo install -d -o "$USER" -g "$(id -gn)" "$CI_DATA_ROOT"
+          mkdir -p "$CI_DATA_ROOT/docker" "$CI_DATA_ROOT/workspace"
           if [ "${{ matrix.mode }}" = "multinode" ]; then
-            sudo rm -rf /mnt/data/docker-node1 /mnt/data/docker-node2
-            sudo cp -a /mnt/data/docker /mnt/data/docker-node1
-            sudo cp -a /mnt/data/docker /mnt/data/docker-node2
+            cp -a "$CI_DATA_ROOT/docker" "$CI_DATA_ROOT/docker-node1"
+            cp -a "$CI_DATA_ROOT/docker" "$CI_DATA_ROOT/docker-node2"
           fi
 
       - name: Set up Docker storage
+        if: runner.environment == 'github-hosted'
         run: |
           echo "::group::Current disk usage"
           df -h
@@ -139,47 +101,38 @@ jobs:
           sudo ln -s /mnt/docker /var/lib/docker
           sudo systemctl start docker
 
-      - uses: docker/setup-buildx-action@v3
-
-      - name: Build outer docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          tags: pwncollege/dojo
-          load: true
-          cache-from: type=local,src=/mnt/cache/dojo-image
-          cache-to: type=local,dest=/mnt/cache/dojo-image,mode=max
-
-      - name: Build test docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: test
-          tags: dojo-test
-          load: true
-          cache-from: type=local,src=/mnt/cache/test-image
-          cache-to: type=local,dest=/mnt/cache/test-image,mode=max
-
       - name: Start the dojo for ${{ matrix.mode }} ${{ matrix.tier }} tests
-        timeout-minutes: 30
+        timeout-minutes: 75
         run: |
           COVERAGE_ARG=""
           if [ "${{ matrix.coverage }}" = "true" ]; then
             COVERAGE_ARG="-C"
           fi
-          ./deploy.sh -g -D /mnt/data/docker -W /mnt/data/workspace ${{ matrix.mode == 'multinode' && '-M' || '' }} $COVERAGE_ARG
+          DOJO_DATA_TMPDIR="$CI_DATA_ROOT" ./deploy.sh -b -g -D "$CI_DATA_ROOT/docker" -W "$CI_DATA_ROOT/workspace" ${{ matrix.mode == 'multinode' && '-M' || '' }} $COVERAGE_ARG
+
+      - name: Build test image
+        run: docker build --tag dojo-test test/
+
+      - name: Verify multinode topology
+        if: matrix.mode == 'multinode'
+        run: docker exec dojo jq -e 'type == "object" and length >= 2' /data/workspace_nodes.json
 
       - name: Run ${{ matrix.mode }} ${{ matrix.tier }} tests
         timeout-minutes: 60
         run: |
           COVERAGE_ARG=""
+          PYTEST_ARGS=(-m "${{ matrix.selection }}")
           if [ "${{ matrix.coverage }}" = "true" ]; then
             COVERAGE_ARG="-C"
           fi
-          ./deploy.sh -g -D /mnt/data/docker -W /mnt/data/workspace ${{ matrix.mode == 'multinode' && '-M' || '' }} $COVERAGE_ARG -N -t -m '${{ matrix.selection }}'
+          if [ "${{ matrix.mode }}" = "multinode" ]; then
+            PYTEST_ARGS+=(-p multinode_guard)
+          fi
+          DOJO_DATA_TMPDIR="$CI_DATA_ROOT" ./deploy.sh -g -D "$CI_DATA_ROOT/docker" -W "$CI_DATA_ROOT/workspace" ${{ matrix.mode == 'multinode' && '-M' || '' }} $COVERAGE_ARG -N -t "${PYTEST_ARGS[@]}"
 
       - name: Upload coverage reports to Codecov
         if: matrix.coverage
-        uses: codecov/codecov-action@v5
+        uses: codecov/codecov-action@v7
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: data/coverage/coverage.xml
@@ -198,40 +151,15 @@ jobs:
           echo "::group::Main node systemd logs"
           docker exec dojo journalctl -u '*' -b || echo "No main outer container..."
           echo "::endgroup::"
-          echo "::group::Main node compose logs"
-          docker exec dojo dojo compose logs || echo "No main outer container..."
-          echo "::endgroup::"
           if [ "${{ matrix.mode }}" = "multinode" ]; then
             echo "::group::Workspace node 1 systemd logs"
             docker exec dojo-node1 journalctl -u '*' -b || echo "No node1 outer container..."
             echo "::endgroup::"
-            echo "::group::Workspace node 1 compose logs"
-            docker exec dojo-node1 dojo compose logs || echo "No node1 outer container..."
-            echo "::endgroup::"
             echo "::group::Workspace node 2 systemd logs"
             docker exec dojo-node2 journalctl -u '*' -b || echo "No node2 outer container..."
             echo "::endgroup::"
-            echo "::group::Workspace node 2 compose logs"
-            docker exec dojo-node2 dojo compose logs || echo "No node2 outer container..."
-            echo "::endgroup::"
           fi
 
-      - name: Prepare the cache
-        if: env.SAVE_CACHE == 'yes'
-        run: |
-          ./deploy.sh -K -g
-          df -h
-          sudo umount /mnt/cache
-          du -sm /mnt/cache.img
-
-      - name: Save docker and workspace cache
-        if: env.SAVE_CACHE == 'yes'
-        uses: actions/cache/save@v4
-        with:
-          path: |
-            /mnt/cache.img
-          key: dojo-cache-${{ matrix.mode }}-${{ github.run_id }}
-
       - name: Final filesystem information
         if: always()
         run: |
@@ -245,17 +173,25 @@ jobs:
     needs: test
     if: github.ref == 'refs/heads/master'
     steps:
+      - uses: actions/checkout@v6
+
+      - name: Free runner disk space
+        if: runner.environment == 'github-hosted'
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+                      /usr/local/share/boost /usr/local/share/powershell \
+                      /usr/share/swift /opt/hostedtoolcache || true
+
+      - uses: cachix/install-nix-action@v31
+
+      - name: Build and import outer NixOS image
+        run: ./nix/build-image.sh pwncollege/dojo:latest
+
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push
-        uses: docker/build-push-action@v6
-        with:
-          push: true
-          tags: pwncollege/dojo:latest
+      - name: Push image
+        run: docker push pwncollege/dojo:latest