ci: enforce migration and release quality gates

Author: reacher-z

.editorconfig

@@ -18,6 +18,13 @@ indent_style = tab
 [*.sh]
 indent_size = 2
 
+[*.py]
+max_line_length = 100
+
+[*.tsv]
+indent_style = tab
+trim_trailing_whitespace = false
+
 # Keep test fixtures byte-exact
 [tests/fixtures/**]
 trim_trailing_whitespace = false

.github/workflows/ci.yml

@@ -15,6 +15,34 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  quality:
+    name: Static quality and data integrity
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: pip
+          cache-dependency-path: requirements*.txt
+      - run: python -m pip install -r requirements-dev.txt
+      - name: Install shell quality tooling
+        run: sudo apt-get update && sudo apt-get install -y shellcheck
+      - name: Python format, lint, and types
+        run: |
+          ruff format --check rewardharness src scripts examples tests vanilla
+          ruff check rewardharness src scripts examples tests vanilla
+          mypy rewardharness
+      - name: Rating and shell integrity
+        run: |
+          python scripts/check_rating_integrity.py
+          python scripts/check_migration_coverage.py
+          pip-audit -r requirements.txt
+          for file in scripts/*.sh scripts/lib/*.sh; do bash -n "$file"; done
+          shellcheck scripts/*.sh scripts/lib/*.sh
+
   test:
     name: Test on Python ${{ matrix.python-version }}
     runs-on: ubuntu-latest
@@ -38,11 +66,11 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -r requirements-dev.txt
 
-      - name: Run test suite
-        run: python -m pytest tests/ -v --tb=short
+      - name: Run test suite with coverage
+        run: python -m pytest tests/ -v --tb=short --cov=rewardharness --cov-report=term-missing
 
       - name: Smoke-test library inspection
-        run: python examples/inspect_library.py
+        run: python -m rewardharness.cli inspect
 
   package:
     name: Validate distribution
@@ -65,6 +93,12 @@ jobs:
           python -m build
           python -m twine check dist/*
 
+      - name: Test wheel installation
+        run: |
+          python -m venv /tmp/rewardharness-wheel
+          /tmp/rewardharness-wheel/bin/pip install dist/*.whl
+          /tmp/rewardharness-wheel/bin/rewardharness inspect
+
       - name: Upload distributions
         uses: actions/upload-artifact@v4
         with:

.gitignore

@@ -15,6 +15,9 @@ build/
 .mypy_cache/
 .ruff_cache/
 .coverage
+.coverage.*
+htmlcov/
+.migration-coverage.json
 .tox/
 
 # Experiment outputs (large; not in public release)

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help install install-dev check test release-check demo evolve benchmark reproduce clean
+.PHONY: help install install-dev check test quality release-check demo evolve benchmark reproduce clean
 
 # Default target: show available commands
 help:
@@ -9,11 +9,12 @@ help:
 	@echo "  make install     Install Python dependencies (core only)               nothing"
 	@echo "  make install-dev Install test and release tooling                      nothing"
 	@echo "  make test        Run the test suite                                    nothing (no GPU/net)"
+	@echo "  make quality     Run lint, types, data integrity, and shell checks     dev dependencies"
 	@echo "  make release-check  Build and validate wheel + source distribution     nothing"
 	@echo "  make check       Preflight: verify env vars / creds / endpoints       nothing (probes only)"
 	@echo "  make demo        1-iter smoke test from examples/seed_library          Gemini + vLLM (or HF)"
 	@echo "  make benchmark   K=2/3/4 accuracy on EditReward-Bench (read-only)     Gemini + vLLM"
-	@echo "  make evolve      Evolve src/library/ for 200 iters → results/my_run/  Gemini + vLLM"
+	@echo "  make evolve      Evolve packaged library for 200 iters → results/my_run/ Gemini + vLLM"
 	@echo "  make reproduce   End-to-end EditReward-Bench reproduction (~4-6 h)    Gemini + 4 GPUs"
 	@echo "  make clean       Remove caches and generated artifacts                 nothing"
 	@echo ""
@@ -24,43 +25,54 @@ help:
 	@echo "See .env.example for the full list."
 
 install:
-	pip install -r requirements.txt
+	python -m pip install -r requirements.txt
 
 install-dev:
-	pip install -r requirements-dev.txt
+	python -m pip install -r requirements-dev.txt
 
 check:
 	python scripts/check_env.py
 
 test:
-	python -m pytest tests/ -v
+	python -m pytest tests/ -v --cov=rewardharness --cov-report=term-missing
 
-release-check:
+quality:
+	ruff format --check rewardharness src scripts examples tests vanilla
+	ruff check rewardharness src scripts examples tests vanilla
+	mypy rewardharness
+	python scripts/check_rating_integrity.py
+	python scripts/check_migration_coverage.py
+	pip-audit -r requirements.txt
+	@for file in scripts/*.sh scripts/lib/*.sh; do bash -n "$$file"; done
+	@if command -v shellcheck >/dev/null 2>&1; then shellcheck scripts/*.sh scripts/lib/*.sh; \
+	else echo "shellcheck not installed; bash syntax checks completed"; fi
+
+release-check: quality test
 	python -m build
 	python -m twine check dist/*
 
 demo:
-	python scripts/run_evolution.py \
+	rewardharness evolve \
 	  --config configs/default.yaml \
 	  --library-dir examples/seed_library \
 	  --results-dir results/demo/ \
 	  --max-iters 1
 
 evolve:
-	python scripts/run_evolution.py \
+	rewardharness evolve \
 	  --config configs/default.yaml \
 	  --results-dir results/my_run/ \
 	  --max-iters 200
 
 benchmark:
-	python scripts/run_benchmark.py \
+	rewardharness benchmark \
 	  --config configs/default.yaml
-# Defaults to the paper-evolved library shipped at src/library/ and reports
+# Defaults to the paper-evolved library shipped as package resources and reports
 # K=2/3/4 accuracy on EditReward-Bench. The paper's headline 45.7% / 47.4%
 # average is mean(K=2, K=3, K=4, GenAI-Bench) — for the GenAI-Bench column
 # you need a separate eval pass (see OUTPUTS.md §"After make benchmark").
 # To benchmark a different library — e.g. one you evolved yourself or the
-# small examples/seed_library/ — call run_benchmark.py directly with
+# small examples/seed_library/ — call the CLI with
 # --library-dir <path>.
 
 reproduce: