ci: replace Python test suite with BATS and consolidate workflows

Replace the pytest/testinfra/tox test stack with BATS, aligning with
the approach used in the FTL repository.

- Merge build-and-test.yml into build-and-publish.yml; the combined
  lint+test job now runs on pull_request via a single bash test/run.sh
  call, removing the need for Python/tox in CI
- Replace Python test files with test/run.sh and test/test_suite.bats
- test/run.sh handles image build, BATS install, container lifecycle,
  and cleanup via trap in one place
- Containers consolidated from 6 to 2 (CONTAINER_DEFAULT and
  CONTAINER_CUSTOM), removing tests that belong to FTL's own suite
- Tests now focus on Docker-specific behaviour: entrypoint, signal
  handling, UID/GID mapping, cron setup, and password assignment

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Signed-off-by: Adam Warner <me@adamwarner.co.uk>

Author: PromoFaux

.github/workflows/build-and-publish.yml

@@ -16,7 +16,39 @@ env:
   components_branch: ${{ github.event_name == 'release' && 'master' || 'development' }}
 
 jobs:
+  test:
+    if: github.event_name == 'pull_request'
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/386
+            runner: ubuntu-latest
+          - platform: linux/arm/v6
+            runner: ubuntu-24.04-arm
+          - platform: linux/arm/v7
+            runner: ubuntu-24.04-arm
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+          - platform: linux/riscv64
+            runner: ubuntu-24.04-arm
+    env:
+      CI_ARCH: ${{ matrix.platform }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a #v4.0.0
+
+      - name: Test
+        run: CIPLATFORM=${{ env.CI_ARCH }} bash test/run.sh
+
   build-prepare:
+    if: github.event_name != 'pull_request'
     runs-on: ubuntu-24.04
     outputs:
       components_branch: ${{ env.components_branch }}
@@ -25,6 +57,7 @@ jobs:
       - run: echo "Exposing env vars for reusable workflow"
 
   build:
+    if: github.event_name != 'pull_request'
     uses: docker/github-builder/.github/workflows/build.yml@v1
     needs:
       - build-prepare
@@ -45,7 +78,7 @@ jobs:
         WEB_BRANCH=${{ needs.build-prepare.outputs.components_branch }}
         PADD_BRANCH=${{ needs.build-prepare.outputs.components_branch }}
       platforms: linux/amd64,linux/386,linux/arm/v6,linux/arm/v7,linux/arm64,linux/riscv64
-      push: ${{ github.event_name != 'pull_request' }}
+      push: true
       set-meta-labels: true
       meta-images: |
         pihole/pihole

.github/workflows/build-and-test.yml

@@ -15,6 +15,7 @@ etc-pihole/
 var-log/
 .vscode/
 .pytest_cache/
+test/libs/
 
 # WIP/test stuff
 doco.yml

.gitignore

@@ -1,12 +1,22 @@
 # Prerequisites
 
-Make sure you have `docker`, `python` and `tox` installed.
+Make sure you have `docker` and `git` installed.
 
 # Running tests locally
 
-`tox -c test/tox.ini`
+```sh
+bash test/run.sh
+```
 
-Should result in:
+This will:
 
-- An image named `pihole:CI_container` being built
-- Tests being ran to confirm the image doesn't have any regressions
+- Build an image named `pihole:test`
+- Start a set of containers (one per configuration under test)
+- Run the BATS test suite against those containers
+- Remove all test containers on exit
+
+To test a specific platform via emulation, set `CIPLATFORM`:
+
+```sh
+CIPLATFORM=linux/arm64 bash test/run.sh
+```

test/TESTING.md

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Run from the test/ directory regardless of where the script is called from
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+# ---- Build the image --------------------------------------------------------
+
+PLATFORM_ARGS=()
+[ -n "${CIPLATFORM:-}" ] && PLATFORM_ARGS=(--platform "${CIPLATFORM}")
+
+docker buildx build \
+    --load \
+    "${PLATFORM_ARGS[@]}" \
+    --progress plain \
+    -f ../src/Dockerfile \
+    -t pihole:test \
+    ../src/
+
+# ---- Install BATS -----------------------------------------------------------
+
+if [ -z "${BATS:-}" ]; then
+    mkdir -p libs
+    if [ ! -d libs/bats ]; then
+        git clone --depth=1 --quiet https://github.com/bats-core/bats-core libs/bats
+    fi
+    BATS=libs/bats/bin/bats
+fi
+
+# ---- Start containers -------------------------------------------------------
+
+# Cleanup all test containers on exit (success or failure)
+CONTAINERS=()
+cleanup() {
+    if [ ${#CONTAINERS[@]} -gt 0 ]; then
+        docker rm -f "${CONTAINERS[@]}" > /dev/null 2>&1 || true
+    fi
+}
+trap cleanup EXIT
+
+start_container() {
+    local id
+    id=$(docker run -d -t "${PLATFORM_ARGS[@]}" -e TZ="Europe/London" "$@" pihole:test)
+    CONTAINERS+=("$id")
+    echo "$id"
+}
+
+CONTAINER_DEFAULT=$(start_container)
+CONTAINER_CUSTOM=$(start_container \
+    -e PIHOLE_UID=456 \
+    -e PIHOLE_GID=456 \
+    -e FTLCONF_webserver_api_password=1234567890)
+
+export CONTAINER_DEFAULT CONTAINER_CUSTOM CIPLATFORM
+
+# ---- Wait for containers to be ready ----------------------------------------
+
+wait_for_ftl() {
+    local container="$1"
+    local timeout=60
+    local elapsed=0
+    printf "Waiting for FTL in %.12s... " "${container}"
+    until docker logs "${container}" 2>&1 | grep -q "########## FTL started"; do
+        sleep 1
+        elapsed=$(( elapsed + 1 ))
+        if (( elapsed >= timeout )); then
+            echo "TIMEOUT"
+            echo "--- Container logs ---"
+            docker logs "${container}"
+            return 1
+        fi
+    done
+    echo "ready (${elapsed}s)"
+}
+
+for container in "$CONTAINER_DEFAULT" "$CONTAINER_CUSTOM"; do
+    wait_for_ftl "$container"
+done
+
+# ---- Run BATS ---------------------------------------------------------------
+
+"$BATS" -p test_suite.bats

test/requirements.txt

@@ -0,0 +1,89 @@
+#!/usr/bin/env bats
+
+# Containers are started by run.sh and their IDs exported as environment
+# variables. All tests (except the shutdown test) share these containers,
+# so each configuration is only booted once per test run.
+#
+# CONTAINER_DEFAULT   - no extra env vars
+# CONTAINER_CUSTOM    - PIHOLE_UID=456, PIHOLE_GID=456, FTLCONF_webserver_api_password=1234567890
+
+# ---- FTL binary -------------------------------------------------------------
+
+@test "FTL reports version" {
+    run docker exec "$CONTAINER_DEFAULT" pihole-FTL -vv
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"Version:"* ]]
+}
+
+@test "FTL reports correct architecture" {
+    [ -n "${CIPLATFORM:-}" ] || skip "CIPLATFORM not set, running locally"
+    run docker exec "$CONTAINER_DEFAULT" pihole-FTL -vv
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"Architecture:"* ]]
+    [[ "$output" == *"$CIPLATFORM"* ]]
+}
+
+@test "FTL starts up and shuts down cleanly" {
+    # This test needs its own container because it stops it
+    local platform_args=()
+    [ -n "${CIPLATFORM:-}" ] && platform_args=(--platform "$CIPLATFORM")
+
+    local container
+    container=$(docker run -d -t "${platform_args[@]}" -e TZ="Europe/London" pihole:test)
+
+    # Wait for FTL to start
+    local timeout=60
+    local elapsed=0
+    until docker logs "$container" 2>&1 | grep -q "########## FTL started"; do
+        sleep 1
+        elapsed=$(( elapsed + 1 ))
+        if (( elapsed >= timeout )); then
+            docker rm -f "$container"
+            echo "FTL did not start within ${timeout}s"
+            return 1
+        fi
+    done
+
+    # Stop gracefully (SIGTERM), then capture logs before removing
+    docker stop "$container"
+    local logs
+    logs=$(docker logs "$container" 2>&1)
+    docker rm "$container"
+
+    [[ "$logs" == *"INFO: ########## FTL terminated after"* ]]
+    [[ "$logs" == *"(code 0)"* ]]
+}
+
+# ---- Container configuration ------------------------------------------------
+
+@test "Cron file is valid" {
+    run docker exec "$CONTAINER_DEFAULT" bash -c \
+        "/usr/bin/crontab /crontab.txt 2>&1; crond -d 8 -L /cron.log 2>&1; cat /cron.log"
+    [[ "$output" != *"parse error"* ]]
+}
+
+@test "Custom PIHOLE_UID is applied to pihole user" {
+    run docker exec "$CONTAINER_CUSTOM" id -u pihole
+    [ "$status" -eq 0 ]
+    [ "$output" = "456" ]
+}
+
+@test "Custom PIHOLE_GID is applied to pihole group" {
+    run docker exec "$CONTAINER_CUSTOM" id -g pihole
+    [ "$status" -eq 0 ]
+    [ "$output" = "456" ]
+}
+
+# ---- Web password setup -----------------------------------------------------
+
+@test "Random password is assigned on fresh start" {
+    run docker logs "$CONTAINER_DEFAULT"
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"assigning random password:"* ]]
+}
+
+@test "Password defined by environment variable is used" {
+    run docker exec "$CONTAINER_CUSTOM" bash -c ". bash_functions.sh; setup_web_password"
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"Assigning password defined by Environment Variable"* ]]
+}