feat: provide integration tests scaffolding

Author: aldbr

.github/actions/pixi-setup/action.yml

@@ -0,0 +1,29 @@
+name: Set up pixi
+description: >
+  Install pixi and the requested environment(s), applying the editable-install workaround for
+  prefix-dev/pixi#3762 before the environment is solved. Shared by ci.yml and integration.yml so
+  the workaround lives in one place.
+
+inputs:
+  environments:
+    description: Space-separated pixi environment name(s) to install (e.g. "py313").
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - uses: prefix-dev/setup-pixi@v0.9.3
+      with:
+        run-install: false
+        post-cleanup: false
+    - name: Workaround for prefix-dev/pixi#3762
+      shell: bash
+      run: |
+        # Editable installs break under pixi in CI; pin to non-editable before the env is solved.
+        sed -i.bak 's@editable = true@editable = false@g' pyproject.toml
+        rm pyproject.toml.bak
+        git diff --stat  # show what the workaround changed
+    - uses: prefix-dev/setup-pixi@v0.9.3
+      with:
+        cache: false
+        environments: ${{ inputs.environments }}

.github/workflows/ci.yml

@@ -38,20 +38,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v6
-      - uses: prefix-dev/setup-pixi@v0.9.3
+      - uses: ./.github/actions/pixi-setup
         with:
-          run-install: false
-          post-cleanup: false
-      - name: Apply workarounds
-        run: |
-          # Workaround for https://github.com/prefix-dev/pixi/issues/3762
-          sed -i.bak 's@editable = true@editable = false@g' pyproject.toml
-          rm pyproject.toml.bak
-          # Show any changes
-          git diff
-      - uses: prefix-dev/setup-pixi@v0.9.3
-        with:
-          cache: false
           environments: ${{ matrix.python-version }}
       - name: Run pytest
         run: pixi run -e ${{ matrix.python-version }} pytest

.github/workflows/integration.yml

@@ -0,0 +1,102 @@
+# Skeleton of the manifest-driven integration workflow (IC-ADR-002 §5; scaffold issue #9).
+#
+# The matrix is generated from tests/integration/stacks.yml — adding a stack, configuration or
+# version is a manifest entry plus files, never a workflow edit. generate_matrix.py emits jobs only
+# for *harmonized* stacks (those with a stacks/<id>/compose.yml); stacks that are seeded in the
+# manifest but not yet harmonized are announced with a `::notice` in the matrix job and produce no
+# job here (so this workflow has zero jobs until the first stack lands).
+#
+# Triggers on any PR that touches the integration suite (so a stack PR self-tests the moment it
+# lands files — no workflow edit), plus manual/reusable dispatch. It is NOT yet gating: until issue
+# #14 wires it into ci.yml, has CI pull prebuilt images from GHCR (instead of building on the
+# runner), and makes it a required check, a red run here does not block merges.
+name: Integration tests
+
+on:
+  workflow_dispatch:
+  workflow_call:
+  pull_request:
+    paths:
+      - "tests/integration/**"
+      - ".github/workflows/integration.yml"
+      - ".github/actions/pixi-setup/**"
+
+permissions:
+  contents: read
+
+jobs:
+  matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.gen.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v6
+      - name: Generate {stack x config x version} matrix from stacks.yml
+        id: gen
+        # Same script developers run locally (without --github-output) to preview the matrix. It
+        # also filters to harmonized stacks and prints a notice for the seeded-only ones.
+        # PyYAML ships with the runner's system python; `pip install pyyaml` here if that changes.
+        run: python3 tests/integration/generate_matrix.py --github-output
+
+  integration:
+    needs: matrix
+    if: needs.matrix.outputs.matrix != '[]'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.matrix.outputs.matrix) }}
+    name: ${{ matrix.stack }} / ${{ matrix.config }} / ${{ matrix.version }}
+    env:
+      COMPOSE_FILE: tests/integration/stacks/${{ matrix.compose_stack }}/compose.yml
+      BACKEND_VERSION: ${{ matrix.version }}
+    steps:
+      - uses: actions/checkout@v6
+
+      # Every job in the matrix is a harmonized stack (the matrix job filtered out the rest), so
+      # compose.yml is guaranteed present — no per-step "has this stack landed?" guard is needed.
+      - name: Start the stack (healthchecks gate readiness)
+        run: docker compose -f "$COMPOSE_FILE" up --wait
+
+      - uses: ./.github/actions/pixi-setup
+        if: matrix.exec_in_container == false
+        with:
+          environments: py313
+
+      - name: Run the contract suite
+        run: |
+          if [ "${{ matrix.exec_in_container }}" = "true" ]; then
+            # local-* stacks: run pytest inside the batch container (IC-ADR-002 §1). The batch
+            # scheduler typically refuses to run as root (HTCondor, Slurm), so the image must ship
+            # an unprivileged test user and the suite + pytest must be mounted or baked in.
+            # TODO(#13/#14): mount/install the suite and pin the exec user (compose `user:` or
+            # `exec --user <test-user>`); see docs/dev/reference/integration-stacks.md.
+            runner=(docker compose -f "$COMPOSE_FILE" exec -T backend pytest)
+          else
+            runner=(pixi run -e py313 pytest)
+          fi
+          set +e  # keep going so exit 5 (no tests selected) can be handled below
+          "${runner[@]}" tests/integration -m "${{ matrix.markers }}" \
+            --stack="${{ matrix.stack }}" --config="${{ matrix.config }}"
+          rc=$?
+          # pytest exits 5 when the stack's markers select no tests. Phase 0 ships only the
+          # manifest tests, so the first harmonized stack would otherwise go red for "no tests
+          # ran"; tolerate exit 5 with a notice until the Phase-2 contract suite lands.
+          if [ "$rc" -eq 5 ]; then
+            echo "::notice::stack '${{ matrix.stack }}': no tests matched markers '${{ matrix.markers }}' (pytest exit 5) - passing until the contract suite lands"
+            rc=0
+          fi
+          exit "$rc"
+
+      - name: Collect daemon logs
+        if: failure()
+        # Daemons run in the foreground and log to stdout/stderr (IC-ADR-002 §3), so compose logs
+        # are the whole failure story — a file-logging daemon redirects its log to /dev/stdout.
+        run: docker compose -f "$COMPOSE_FILE" logs --no-color > compose-logs.txt || true
+
+      - name: Upload logs as artifacts
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-${{ matrix.stack }}-${{ matrix.config }}-${{ matrix.version }}
+          path: compose-logs.txt

.gitignore

@@ -76,6 +76,9 @@ htmlcov/
 #remove in case we want a specific LHCb one
 .pylintrc
 
+# Integration-stack credentials: minted per run into a bind mount, never committed (IC-ADR-002 §4)
+tests/integration/stacks/*/credentials/
+
 # docs (auto-generated build output only; docs/assets is tracked)
 docs/_build/
 site/

RENOVATE.md

@@ -0,0 +1,3 @@
+# Explanations
+
+- [Why integration stacks — testing against real backend daemons](integration-testing.md)

docs/dev/explanations/index.md

@@ -0,0 +1,38 @@
+# Why integration stacks — testing against real backend daemons
+
+The full reasoning is [IC-ADR-002](../../adr/IC-ADR-002_integration_tests.md); this page is the
+short version plus the phasing plan.
+
+## Why real daemons
+
+interCEde's architecture is deliberately mock-friendly — everything is a `Protocol`, and the unit
+suite exercises contracts against fakes. That is also its greatest testing risk: a structural
+protocol and a fake will happily agree with each other forever while the real ARC REST endpoint
+or the real `condor_submit -spool` handshake drift away underneath. The bugs interCEde exists to
+absorb (version-specific daemon behaviour, destructive output retrieval, status-mapping quirks)
+are exactly the bugs unit tests structurally cannot see — HTCondor ≥ 25.8 silently turning
+spooled-output retrieval into a one-shot operation is the canonical example.
+
+So the integration suite runs **real backend daemons** in containerised **stacks**, on every pull
+request, and every backend version is pinned and tracked by Renovate so an upstream release that
+breaks a contract surfaces as a red, bisectable PR.
+
+## Capability symmetry
+
+The integration suite is the executable form of the contract: it narrows to optional capabilities
+exactly the way the library's consumers do (`isinstance` against a `runtime_checkable` protocol)
+and skips where a backend genuinely lacks a capability. If a capability check works in the test
+harness, it works for DiracX; if a backend's structural typing lies, a real daemon is where the lie
+is caught. The skip helper that does this lands with the Phase-2 contract suite that calls it —
+its signature would otherwise be guessed ahead of its callers.
+
+## Phase 0 vs Phase 2
+
+**Phase 0 (now):** the `intercede` package's protocols don't exist yet, so stack PRs validate
+their stacks with **native client tools** (`arcsub`/`arcstat`/`arcget`, `condor_submit`,
+`sbatch`) in marker-selected test modules.
+
+**Phase 2:** a single backend-agnostic **contract suite** (`test_submit.py`, `test_status.py`,
+`test_output.py`, `test_kill.py`) written against the interCEde protocols replaces the
+native-tool tests. Backend-specific semantics (e.g. HTCondor's destructive fetch) get dedicated
+marker-selected tests (`destructive_fetch`) rather than conditionals inside generic ones.

docs/dev/explanations/integration-testing.md

@@ -0,0 +1,52 @@
+# Add an integration stack
+
+A *stack* is a named, self-contained backend environment (one `docker compose` file plus
+versioned configuration) that the integration suite runs against. Adding one requires **no
+workflow edits** — the CI matrix is generated from the manifest.
+
+Before starting, read the [stack contract](../reference/integration-stacks.md) — the rules below
+reference it.
+
+## Steps
+
+1. **Copy the template**: `tests/integration/stacks/_template/` →
+   `tests/integration/stacks/<id>/`. Fill in `compose.yml` (real healthcheck, host-readable
+   `./credentials` bind mount) and `config/basic/` (backend config + `intercede-client.toml`,
+   which change together as one atomic configuration). An **aliased** stack (`stack:` reusing
+   another's compose) ships only `config/` under its own id — no `compose.yml` of its own — and its
+   `markers` exclude the borrowed transport (e.g. `and not ssh`).
+2. **Add the image** under `tests/integration/stacks/_images/<name>/` following the
+   [image conventions](../reference/integration-images.md) (EL9, pinned Renovate-annotated
+   `ARG`s, foreground daemons, ephemeral credentials at start).
+3. **Add the manifest entry** in `tests/integration/stacks.yml` (`id`, `configs`, `markers`,
+   optionally `versions` / `stack:` alias / `exec_in_container`).
+4. **Verify locally**:
+
+   ```sh
+   docker compose -f tests/integration/stacks/<id>/compose.yml up --wait
+   pixi run -e py313 pytest tests/integration -m "<markers>" --stack=<id> --config=basic
+   pixi run -e py313 pytest tests/integration/test_manifest.py   # layout checks
+   ```
+
+## Harmonization checklist
+
+For a stack PR (including the in-flight prototype stacks converging onto this layout):
+
+- [ ] Files under `stacks/<id>/` following `_template/` (compose.yml + `config/basic/`)
+- [ ] `docker compose … up --wait` succeeds locally; healthchecks probe real readiness
+- [ ] Daemons foregrounded (no systemd), logging to stdout/stderr; Docker (not Podman) on
+      EL9-based images
+- [ ] Dockerfile under `stacks/_images/<name>/` with pinned, `# renovate:`-annotated `ARG`s
+- [ ] Credentials generated at start, world-readable, into the host-readable `./credentials` bind
+      mount under `credentials/<id>/…` (gitignored)
+- [ ] `intercede-client.toml` filled in (own `id`, endpoint, credential type/path, queue); a
+      host-run stack publishes its port on localhost with a matching Test-CA SAN
+- [ ] `stacks.yml` entry present; `pytest tests/integration/test_manifest.py` green
+- [ ] At least one marker-selected Phase-0 smoke test ships with the stack (so its job isn't just
+      "no tests ran" — the workflow tolerates that, but a smoke test makes the job meaningful)
+- [ ] Bespoke per-PR workflow removed: the manifest-driven `integration.yml` now runs
+      automatically on any PR touching `tests/integration/**`, so the stack self-tests. It stays
+      **non-blocking** until issue #14 (GHCR image pulls + `ci.yml` invocation + required check),
+      so keep a bespoke workflow only if you need a *gating* check in the interim
+- [ ] `docker compose logs` captures the daemons' output on failure (daemons log to stdout; a
+      file-logging daemon redirects to `/dev/stdout`)

docs/dev/how-to/add-an-integration-stack.md

@@ -0,0 +1,4 @@
+# How-to guides
+
+- [Run the integration tests locally](run-integration-tests-locally.md)
+- [Add an integration stack](add-an-integration-stack.md)

docs/dev/how-to/index.md

@@ -0,0 +1,56 @@
+# Run the integration tests locally
+
+Everything CI does is reproducible on a laptop — a design rule of
+[IC-ADR-002](../../adr/IC-ADR-002_integration_tests.md): CI does nothing a developer cannot do
+locally.
+
+## Run one stack
+
+1. Start the stack; healthchecks gate readiness, so when `up --wait` returns the backend is
+   usable:
+
+   ```sh
+   docker compose -f tests/integration/stacks/<id>/compose.yml up --wait
+   ```
+
+2. Run the applicable test subset — the stack's `markers:` expression is in
+   `tests/integration/stacks.yml`:
+
+   ```sh
+   pixi run -e py313 pytest tests/integration -m "<markers>" --stack=<id> --config=basic
+   ```
+
+For `local-*` stacks (`exec_in_container: true`) the tests run *inside* the batch container
+instead:
+
+```sh
+docker compose -f tests/integration/stacks/<compose-stack>/compose.yml exec --user <test-user> backend \
+  pytest tests/integration -m "<markers>" --stack=<id> --config=basic
+```
+
+Two conditions this recipe depends on (both open TODOs, #13/#14): the image must ship the suite
+and pytest (mounted or baked in), and it must run as an **unprivileged** user — HTCondor and Slurm
+refuse to run jobs as root, hence `--user <test-user>` (or a compose `user:`). Note `--stack`
+still names the aliased entry (e.g. `local-slurm`) so its own `intercede-client.toml` is used.
+
+Stacks double as development environments: leave the stack up and point your client config at
+`localhost` to develop against a real backend.
+
+## Preview the CI matrix
+
+The CI matrix is generated from `stacks.yml` by a plain script — run it locally to see exactly
+which `{stack × config × version}` jobs CI would create:
+
+```sh
+pixi run -e py313 python tests/integration/generate_matrix.py
+```
+
+(The workflow calls the same script with `--github-output`.)
+
+## Manifest-only checks (no Docker needed)
+
+The manifest schema and stack-layout checks run as ordinary unit tests:
+
+```sh
+pixi run -e py313 pytest tests/integration/test_manifest.py
+```