bench: scaffold Tier-B agent-loop token study — ready to run (roadmap… #1309
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main, "claude/**"] | |
| pull_request: | |
| # Cancel superseded runs on the same ref. | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| lint: | |
| name: Lint & type-check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.14" | |
| enable-cache: true | |
| - run: uv sync --frozen | |
| - name: ruff (lint) | |
| run: uv run ruff check . | |
| - name: ruff (format) | |
| run: uv run ruff format --check . | |
| - name: mypy | |
| run: uv run mypy src/mcpg | |
| security: | |
| name: Security audit (SAST & Dependencies) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.14" | |
| enable-cache: true | |
| - run: uv sync --frozen | |
| # pip-audit consults the PyPI advisory DB + OSV.dev for known | |
| # CVEs in resolved dependencies. ``--strict`` upgrades warnings | |
| # to failures so a vulnerable transitive dep blocks merge. We | |
| # audit the uv-resolved requirements (runtime deps only, project | |
| # excluded — the editable install of mcpg itself isn't on PyPI). | |
| - name: Dependency audit (pip-audit) | |
| run: | | |
| uv export --no-dev --no-emit-project --format requirements-txt > /tmp/mcpg-reqs.txt | |
| uv run pip-audit --strict --disable-pip -r /tmp/mcpg-reqs.txt | |
| - name: Bandit static analysis (SAST) | |
| run: uv run bandit -r src/mcpg --skip B101,B608,B110 -ll | |
| test: | |
| name: Tests (PG ${{ matrix.postgres }}) | |
| runs-on: ubuntu-latest | |
| # PG 19 is beta. We track it as an experimental matrix entry — | |
| # failures are reported but don't fail the job. Promoted to | |
| # required once PG 19 reaches GA (issue #120, Phase 3). | |
| continue-on-error: ${{ matrix.experimental == true }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| postgres: ["14", "15", "16", "17", "18"] | |
| include: | |
| - postgres: "19" | |
| experimental: true | |
| # WarehousePG (Greenplum-derived MPP) characterisation lane. | |
| # Pinned to woblerr/warehousepg:7.4.1-WHPG — see | |
| # .github/ci-postgres-warehousepg.Dockerfile (single source of | |
| # truth for the upstream image tag) and roadmap row 15.8. | |
| # `continue-on-error: true` stays on: it's a single community | |
| # image with no fallback, and a single-node Greenplum-family | |
| # cluster is inherently slower/heavier to bring up in CI than a | |
| # plain postgres container. | |
| - postgres: "warehousepg-latest" | |
| experimental: true | |
| steps: | |
| - uses: actions/checkout@v7 | |
| # The ubuntu-latest runner ships an older PostgreSQL client by | |
| # default; `pg_dump` refuses to dump from a newer server (PG 17 / | |
| # 18 in this matrix). Install the matching client tools from the | |
| # pgdg apt repo so `pg_dump` matches the matrix server version | |
| # exactly. Required by the Phase-24 subprocess integration tests | |
| # (mcpg.shell + dump_database); harmless for the older versions. | |
| - name: Install PostgreSQL ${{ matrix.postgres }} client tools | |
| # WarehousePG is wire-compatible with libpq, so the default | |
| # runner-shipped psql works. We don't try to install a custom | |
| # `postgresql-client-warehousepg` package — it doesn't exist on | |
| # pgdg and the existing client is what mcpg.shell expects. | |
| if: matrix.postgres != 'warehousepg-latest' | |
| run: | | |
| sudo apt-get install -y wget gnupg lsb-release ca-certificates | |
| wget --quiet -O- https://www.postgresql.org/media/keys/ACCC4CF8.asc \ | |
| | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg | |
| echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | |
| | sudo tee /etc/apt/sources.list.d/pgdg.list | |
| sudo apt-get update | |
| # PG 19 client packages are published under `postgresql-client-19` | |
| # by the pgdg repo as soon as the server enters beta — but during | |
| # the beta window the package can be temporarily absent from the | |
| # noble suite (observed mid-Beta-1, 2026-06-22). Probe apt-cache | |
| # for the literal "Unable to locate package" failure mode before | |
| # installing so the step distinguishes "package not yet published" | |
| # (downgraded to a workflow ::warning::) from genuine apt errors | |
| # (network / mirror outage — kept fatal so we don't silently mask | |
| # real infra problems; sourcery feedback on PR #152). The matrix | |
| # entry is already `continue-on-error: true` for the | |
| # `Tests (PG 19)` job; this just stops the step-level webhook | |
| # noise. See roadmap 14.1 in docs/feature-shortlist.md. | |
| set +e | |
| pkg="postgresql-client-${{ matrix.postgres }}" | |
| apt_cache_out=$(apt-cache show "$pkg" 2>&1) | |
| apt_cache_rc=$? | |
| set -e | |
| if [ "$apt_cache_rc" -eq 0 ]; then | |
| sudo apt-get install -y "$pkg" | |
| elif printf '%s' "$apt_cache_out" | grep -q "Unable to locate package\|No packages found"; then | |
| echo "::warning::$pkg not yet in apt cache — skipping. Tests that shell out to pg_dump will be limited to what the runner's default client provides." | |
| else | |
| echo "::error::apt-cache show $pkg failed with an unexpected error (likely a transient network / mirror issue, not a package-availability one):" | |
| printf '%s\n' "$apt_cache_out" >&2 | |
| exit "$apt_cache_rc" | |
| fi | |
| # A custom image (pgvector + PostGIS) so the integration suite can test | |
| # vector and geospatial features against a real database. It is built | |
| # and run as a step because GitHub `services:` cannot build images. | |
| - name: Start PostgreSQL (pgvector + PostGIS) | |
| run: | | |
| # Docker Hub intermittently times out resolving the | |
| # pgvector base-image manifest (observed across the matrix: | |
| # "Head https://registry-1.docker.io/...: i/o timeout"). That | |
| # is transient infra, not a code failure, so retry the build | |
| # with exponential backoff before giving up. | |
| # PG 19 (beta) needs a separate Dockerfile — pgvector doesn't | |
| # publish a `pg19` image tag yet, so we build pgvector from source | |
| # against `postgres:19beta1`. Drop this branch once pgvector ships | |
| # a PG 19 image (issue #120 Phase 3, on GA). | |
| DOCKERFILE=".github/ci-postgres.Dockerfile" | |
| if [ "${{ matrix.postgres }}" = "19" ]; then | |
| DOCKERFILE=".github/ci-postgres-pg19.Dockerfile" | |
| elif [ "${{ matrix.postgres }}" = "warehousepg-latest" ]; then | |
| # WarehousePG (Greenplum-derived MPP) lane — see roadmap 15.8 | |
| # and .github/ci-postgres-warehousepg.Dockerfile. The matrix | |
| # entry is `continue-on-error: true` so an upstream image | |
| # outage doesn't block the main suite. | |
| DOCKERFILE=".github/ci-postgres-warehousepg.Dockerfile" | |
| fi | |
| build_db() { | |
| docker build -f "$DOCKERFILE" \ | |
| --build-arg PG_MAJOR=${{ matrix.postgres }} -t mcpg-ci-db . | |
| } | |
| attempt=1 | |
| max_attempts=4 | |
| until build_db; do | |
| if [ "$attempt" -ge "$max_attempts" ]; then | |
| echo "docker build failed after $attempt attempts" >&2 | |
| exit 1 | |
| fi | |
| backoff=$((2 ** attempt)) | |
| echo "docker build attempt $attempt failed; retrying in ${backoff}s..." >&2 | |
| sleep "$backoff" | |
| attempt=$((attempt + 1)) | |
| done | |
| # -e POSTGRES_* is inert for the WarehousePG lane — that image's | |
| # entrypoint reads GREENPLUM_*-prefixed vars instead (baked into | |
| # ci-postgres-warehousepg.Dockerfile), so its superuser is | |
| # `gpadmin`, not `postgres`. Harmless to still pass these; every | |
| # other matrix entry needs them. | |
| docker run -d --name mcpg-db -p 5432:5432 \ | |
| -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres \ | |
| -e POSTGRES_DB=mcpg_test mcpg-ci-db | |
| if [ "${{ matrix.postgres }}" = "warehousepg-latest" ]; then | |
| DB_USER=gpadmin | |
| # A single-node Greenplum-family coordinator+segment cluster | |
| # takes much longer to initialize than a plain postgres | |
| # container, so this lane gets a longer, coarser poll. | |
| max_attempts=90 | |
| sleep_secs=4 | |
| else | |
| DB_USER=postgres | |
| max_attempts=30 | |
| sleep_secs=2 | |
| fi | |
| echo "DB_USER=$DB_USER" >> "$GITHUB_ENV" | |
| for _ in $(seq 1 "$max_attempts"); do | |
| docker exec mcpg-db pg_isready -U "$DB_USER" && break | |
| sleep "$sleep_secs" | |
| done | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.14" | |
| enable-cache: true | |
| - run: uv sync --frozen | |
| # Runs unit, vendored-kernel, and integration suites with the coverage | |
| # gate (fail_under = 90, authored code only) against each PG version. | |
| - name: pytest | |
| env: | |
| MCPG_TEST_DATABASE_URL: postgresql://${{ env.DB_USER }}:postgres@localhost:5432/mcpg_test | |
| run: uv run pytest -q --cov |