Finish pytest-owned test architecture and cut lane runtime (#1316)

* refactor(tests): finish pytest-owned suite architecture

* fix(tests): include exhaustive lane in full CI reproduction

* fix(ci): avoid full benchmarks for root Makefile changes

* fix(ci): release superseded runs before aggregate gates

Author: morluto

.github/workflows/ci.yml

@@ -35,7 +35,6 @@ jobs:
           package-manager-cache: false
       - run: make lint-full
       - run: make typecheck
-      - run: make test-architecture
       - run: make architecture
       - run: make todo-check
       - run: make import-contracts
@@ -62,12 +61,12 @@ jobs:
             timeout-minutes: 15
           - lane: domain
             timeout-minutes: 15
+          - lane: provider
+            timeout-minutes: 25
           - lane: composition
             timeout-minutes: 25
           - lane: e2e
             timeout-minutes: 15
-          - lane: provider
-            timeout-minutes: 25
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -196,23 +195,6 @@ jobs:
         uses: ./.github/actions/setup-lean
       - run: make test-lean
 
-  optional-providers:
-    name: Optional providers
-    if: >-
-      github.event_name != 'pull_request' ||
-      contains(github.event.pull_request.labels.*.name, 'ci:provider') ||
-      contains(github.event.pull_request.labels.*.name, 'ci:full')
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-        with:
-          persist-credentials: false
-      - uses: ./.github/actions/setup-python-tests
-        with:
-          python-version: "3.12"
-      - run: make test-provider
-
   coverage:
     name: Coverage
     if: ${{ always() && !cancelled() }}
@@ -296,8 +278,8 @@ jobs:
 
   required:
     name: required
-    if: ${{ always() }}
-    needs: [static, python, boundaries, wheel, coverage, lean, optional-providers]
+    if: ${{ always() && !cancelled() }}
+    needs: [static, python, boundaries, wheel, coverage, lean]
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
@@ -309,7 +291,6 @@ jobs:
           WHEEL_RESULT: ${{ needs.wheel.result }}
           COVERAGE_RESULT: ${{ needs.coverage.result }}
           LEAN_RESULT: ${{ needs.lean.result }}
-          PROVIDERS_RESULT: ${{ needs.optional-providers.result }}
         run: |
           test "$STATIC_RESULT" = success
           test "$PYTHON_RESULT" = success
@@ -320,14 +301,10 @@ jobs:
             success|skipped) ;;
             *) exit 1 ;;
           esac
-          case "$PROVIDERS_RESULT" in
-            success|skipped) ;;
-            *) exit 1 ;;
-          esac
 
   python-test:
     name: Python Tests
-    if: ${{ always() }}
+    if: ${{ always() && !cancelled() }}
     needs: [required]
     runs-on: ubuntu-latest
     timeout-minutes: 5
@@ -339,7 +316,7 @@ jobs:
 
   lean-test:
     name: Lean Tests
-    if: ${{ always() }}
+    if: ${{ always() && !cancelled() }}
     needs: [lean]
     runs-on: ubuntu-latest
     timeout-minutes: 5
@@ -355,7 +332,7 @@ jobs:
 
   deployment-test:
     name: Deployment Tests
-    if: ${{ always() }}
+    if: ${{ always() && !cancelled() }}
     needs: [static]
     runs-on: ubuntu-latest
     timeout-minutes: 5

.github/workflows/scheduled-validation.yml

@@ -25,6 +25,9 @@ jobs:
       - run: make test-stress
         env:
           PYTEST_ARGS: --durations=20
+      - run: make test-exhaustive
+        env:
+          PYTEST_ARGS: --durations=20
 
   ordering:
     name: Ordering ${{ matrix.lane }} seed ${{ matrix.seed }}

CONTRIBUTING.md

@@ -35,10 +35,10 @@ pre-push hook stays `make lint typecheck`. Focused debugging uses
 `uv run pytest path/to/test.py`. Default `uv run pytest` collects the ordinary
 Lean-free `testpaths`; it does not run storage, process, MCP, or Lean trees.
 
-CI always runs that ordinary Python surface plus storage/process/MCP
-boundaries and the wheel smoke. Lean and optional native/formal providers run
-on merge/main or with the `ci:lean` / `ci:provider` / `ci:full` labels. You do
-not need to reproduce those locally for a routine change.
+CI always runs that ordinary Python surface plus storage/process/MCP,
+maintained Python provider boundaries, and the wheel smoke. Lean runs on
+merge/main or with the `ci:lean` / `ci:full` labels. You do not need to
+reproduce those locally for a routine change.
 
 Specialist lanes (`make test-lean`, `make test-provider`, `make test-storage`,
 `make test-process`, `make test-mcp`, `make test-e2e`, `make test-domain`, and
@@ -230,15 +230,16 @@ Test directories define semantic ownership: `tests/unit`, `tests/component`,
 `tests/domain`, `tests/composition`, `tests/boundary`, and `tests/e2e`. Use the
 matching `make test-*` target as the canonical entry point. Markers are retained
 only when they alter execution: `requires_provider(name)`, `performance`,
-`property`, and `destructive_process`. They do not replace directory ownership.
+`property`, `exhaustive`, and `destructive_process`. They do not replace
+directory ownership. Scheduled validation owns `make test-exhaustive`; keep a
+representative behavioral case in the ordinary owning lane.
 
 Lane execution follows the test directory layout. Storage, process, MCP, and
 Lean stay on named Make targets because they need serial SQLite or kill-safe
 process supervision. Prefer the hydration ladder in the
 [testing strategy](docs/reference/testing-strategy.md): domain services before
 `attached_complete_runtime` before `authorized_complete_runtime` before
-`fresh_complete_runtime`. Inventory complete-runtime usage with
-`make test-runtime-inventory`.
+`fresh_complete_runtime`.
 
 Tests may reuse concept-specific helpers under `tests/support`, but must not
 import helpers from a sibling semantic lane. Keep fixtures in the narrowest

Makefile

@@ -35,7 +35,7 @@ test-unit: ## Pure contracts and models (sequential, 10s).
 		$(PYTEST_DIAGNOSTIC_ARGS) $(PYTEST_ARGS)
 
 test-component: ## One-service component tests (4 workers, 30s).
-	$(UV_RUN) pytest -n 4 --dist worksteal --timeout=30 \
+	$(UV_RUN) pytest -n 4 --dist worksteal --timeout=30 -m "not exhaustive" \
 		$(if $(TESTS),$(TESTS),tests/component) \
 		$(PYTEST_DIAGNOSTIC_ARGS) $(PYTEST_ARGS)
 
@@ -95,6 +95,7 @@ test-compatibility: ## Supported-version import/API compatibility smoke.
 test-all-ci: ## Explicitly run every semantic lane locally (exceptional).
 	$(MAKE) test-unit
 	$(MAKE) test-component
+	$(MAKE) test-exhaustive
 	$(MAKE) test-domain
 	$(MAKE) test-composition
 	$(MAKE) test-storage
@@ -109,6 +110,11 @@ test-stress: ## Repeat explicitly marked property tests on the scheduled lane.
 		--count=$(STRESS_COUNT) $(if $(TESTS),$(TESTS),tests) \
 		$(PYTEST_DIAGNOSTIC_ARGS) $(PYTEST_ARGS)
 
+test-exhaustive: ## Broad finite reference sweeps reserved for scheduled validation.
+	$(UV_RUN) pytest -n 0 --timeout=180 --timeout-method=thread -m exhaustive \
+		$(if $(TESTS),$(TESTS),tests) \
+		$(PYTEST_DIAGNOSTIC_ARGS) $(PYTEST_ARGS)
+
 test-ordering: ## Reproduce scheduled ordering (default seed 17; override with PYTEST_ARGS).
 	@test -n "$(ORDERING_LANE)" || { echo "ORDERING_LANE is required" >&2; exit 2; }
 	$(MAKE) test-$(ORDERING_LANE) \
@@ -147,7 +153,7 @@ precommit: ## Fix and run every routine local handoff check.
 	$(MAKE) fix
 	$(MAKE) quick
 
-check-static: lint-full typecheck test-architecture import-contracts test-runtime-inventory architecture todo-check build ## CI-owned static checks plus a local package build.
+check-static: lint-full typecheck import-contracts architecture todo-check build ## CI-owned static checks plus a local package build.
 
 clean: ## Remove local caches, build outputs, and coverage artifacts.
 	rm -rf .pytest_cache .mypy_cache .ruff_cache dist build htmlcov

benchmarks/tooling/validation_plan.py

@@ -53,6 +53,10 @@ def _discover_conjecture_host_tests() -> tuple[str, ...]:
         "benchmarks/validation/test_heldout_bundle.py",
         "benchmarks/validation/test_heldout_runner.py",
     ),
+    # The root Makefile is parsed by the benchmark contract targets, but its
+    # product-test targets do not alter host verifier behavior. Harbor-owned
+    # execution commands live in make/harbor.mk below.
+    "Makefile": (),
     "benchmarks/tooling/benchmark_timings.py": (
         "benchmarks/validation/test_benchmark_timings.py",
     ),
@@ -92,7 +96,7 @@ def _discover_conjecture_host_tests() -> tuple[str, ...]:
     ),
 }
 SHARED_HOST_HARNESS_PATHS = {
-    "Makefile",
+    "make/harbor.mk",
     "tools/pytest_lifecycle.py",
 }
 

benchmarks/validation/test_benchmark_planner.py

@@ -207,6 +207,29 @@ def test_non_host_control_utilities_do_not_select_full_verifier_corpus(
     _assert_plan_valid(result)
 
 
+def test_root_makefile_runs_contracts_without_host_verifier_replay() -> None:
+    result = planner.plan(["Makefile"], event="pull_request")
+
+    assert result["run-benchmark-check"] == "true"
+    assert result["run-benchmark-record-schema"] == "true"
+    assert result["run-benchmark-host-validation"] == "false"
+    assert _host_matrix(result) == []
+    _assert_plan_valid(result)
+
+
+def test_harbor_makefile_change_keeps_full_host_verifier_coverage() -> None:
+    result = planner.plan(["make/harbor.mk"], event="pull_request")
+
+    assert result["run-benchmark-host-validation"] == "true"
+    assert len(_host_matrix(result)) == 4
+    assert any(
+        "shared verifier execution harness requires full host validation: "
+        "make/harbor.mk" in reason
+        for reason in json.loads(result["benchmark-plan-reasons"])
+    )
+    _assert_plan_valid(result)
+
+
 def test_execution_configuration_change_defers_oracle_to_merge_queue() -> None:
     path = "benchmarks/config/jacobian.mcp.json"
 

benchmarks/validation/test_validation_plan.py

@@ -102,14 +102,22 @@ def test_unknown_tooling_still_falls_back_to_full_validation(tmp_path: Path) ->
     )
 
 
-def test_makefile_change_remains_fail_closed_without_hunk_ownership(
+def test_root_makefile_change_relies_on_contract_gate_not_host_verifiers(
     tmp_path: Path,
 ) -> None:
     plan = host_validation_plan(tmp_path, ["Makefile"], {})
 
+    assert plan.entries == ()
+    assert plan.reasons == ()
+
+
+def test_harbor_makefile_change_remains_fail_closed(tmp_path: Path) -> None:
+    plan = host_validation_plan(tmp_path, ["make/harbor.mk"], {})
+
     assert len(plan.entries) == 4
     assert plan.reasons == (
-        "shared verifier execution harness requires full host validation: Makefile",
+        "shared verifier execution harness requires full host validation: "
+        "make/harbor.mk",
     )
 
 

docs/reference/testing-strategy.md

@@ -77,6 +77,7 @@ make test-mcp
 make test-provider
 make test-lean
 make test-e2e
+make test-exhaustive
 make quick
 make check
 make check-external
@@ -88,6 +89,12 @@ Before starting it, confirm that no other pytest job from this checkout is
 running. Concurrent runtime/store/subprocess suites can turn per-test
 timeouts into host-contention noise.
 
+Broad finite reference sweeps that are valuable but disproportionate for pull
+requests use the `exhaustive` marker. The component lane excludes them, and
+scheduled validation owns `make test-exhaustive`. Keep a representative
+behavioral case in the ordinary owning lane; do not use the marker to defer
+boundary, authorization, persistence, or public-API coverage.
+
 ## Test principles
 
 Tests exercise public behavior and stable artifacts rather than private helper
@@ -105,6 +112,11 @@ contract better than examples. Use representative storage and process seams
 when the claim depends on SQLite, filesystem publication, subprocesses, or
 cancellation.
 
+Let pytest own collection and fixture lifetime through the narrowest owning
+`conftest.py`. Broaden fixture scope only for reusable installation state whose
+tests do not mutate it; tests that revoke authority, patch shared services, or
+otherwise change runtime state retain an isolated fixture.
+
 Do not substitute source-reading tests for caller-visible behavior. If a
 behavioral regression proof is infeasible, state the proof gap.
 

make/development.mk

@@ -1,4 +1,4 @@
-.PHONY: uv-version-check setup doctor setup-lean doctor-lean doctor-external setup-agent container-image eval-image eval-image-pull eval-image-bind deploy-check hooks fix lint complexity-check lint-full security-audit typecheck test-architecture test-runtime-inventory architecture docs-command-check docs-linkcheck
+.PHONY: uv-version-check setup doctor setup-lean doctor-lean doctor-external setup-agent container-image eval-image eval-image-pull eval-image-bind deploy-check hooks fix lint complexity-check lint-full security-audit typecheck architecture docs-command-check docs-linkcheck
 
 uv-version-check: ## Require the repository-pinned uv release.
 	@test "$$(uv --version | awk '{print $$2}')" = "$$(tr -d '[:space:]' < .uv-version)" || { echo "install uv $$(tr -d '[:space:]' < .uv-version) before using this checkout" >&2; exit 2; }
@@ -66,15 +66,9 @@ security-audit: ## Audit dependencies for known vulnerabilities.
 typecheck: ## Run strict static type checking.
 	$(UV_RUN) mypy
 
-test-architecture: ## Enforce semantic test-layer and provider-import boundaries.
-	$(UV_RUN) python -m tools.check_test_architecture .
-
 import-contracts: ## Enforce declared package dependency direction.
 	$(UV_RUN) lint-imports
 
-test-runtime-inventory: ## Fail when authorized complete-runtime uses lack verify/authority signals.
-	$(UV_RUN) python -m tools.inventory_test_runtime --fail-on-unjustified
-
 architecture: ## Enforce product source boundary invariants (subprocess, shutil.which, environ, contracts, surfaces).
 	$(UV_RUN) python tools/check_architecture.py
 

pyproject.toml

@@ -93,6 +93,7 @@ markers = [
     "requires_provider(name): test requires a production provider readiness probe",
     "performance: benchmark or timing evidence excluded from correctness lanes",
     "property: property-based invariant tests",
+    "exhaustive: broad finite reference sweeps run by scheduled validation",
     "destructive_process: test requires child-process isolation and group termination",
 ]