test-pg_search.yml

# workflows/test-pg_search.yml
#
# Test pg_search
# Run unit and integration tests for the pg_search extension.

name: Test pg_search

on:
  # We run nightly to update the code coverage baseline without excessive post-PR runs
  schedule:
    - cron: "0 7 * * *" # daily 07:00 UTC (~2am ET)
  pull_request:
    types: [opened, synchronize, reopened]
    branches:
      - main
      - 0.*.x # Release branches
      - moe/** # Temporary: trigger CI on stacked PR chain
    paths:
      - "Cargo.toml"
      - "Cargo.lock"
      - ".github/workflows/test-pg_search.yml"
      - "pg_search/**"
      - "!pg_search/README.md"
      - "tests/**"
      - "tokenizers/**"
  workflow_dispatch:

concurrency:
  group: test-pg_search-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:
  set-matrix:
    name: Define the PostgreSQL Version Matrix
    runs-on: ubuntu-latest
    if: ${{ !cancelled() }}
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Define the PostgreSQL Version Matrix
        id: set-matrix
        run: |
          if [[ ${{ github.event_name }} == "push" ]]; then
            echo "Push event detected; using only PostgreSQL version 18."
            echo "matrix=[18]" >> $GITHUB_OUTPUT
          else
            echo "Pull request or schedule event detected, using all PostgreSQL versions."
            echo "matrix=[15, 16, 17, 18]" >> $GITHUB_OUTPUT
          fi

  test-pg_search:
    name: Test pg_search on PostgreSQL ${{ matrix.pg_version }} (${{ matrix.pg_impl }} - ${{ matrix.arch }})
    # Compute-optimized c-family — Rust + Postgres tests are CPU-bound.
    # See https://runs-on.com/configuration/job-labels/ — runs in us-east-1.
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=${{ matrix.runner_family }}
      - cpu=${{ matrix.cpu }}
      - ram=${{ matrix.ram }}
      - image=${{ matrix.runner_image }}
      - volume=40gb # default runner ships with 30GB — see https://runs-on.com/runners/linux/
      - extras=s3-cache
    if: ${{ !cancelled() }}
    needs: set-matrix
    strategy:
      fail-fast: false
      matrix:
        # Base: system Postgres on ARM Ubuntu 24.04 for all versions.
        pg_version: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
        pg_impl: [system]
        cpu: [8]
        ram: [16]
        runner_family: [c7g+c8g]
        runner_image: [ubuntu24-full-arm64]
        arch: [arm64]
        include:
          # pgrx compiles Postgres from source with --enable-cassert and --enable-debug,
          # which enables C-level Assert() checks that catch memory corruption, invalid
          # internal state, and broken invariants. System Postgres packages from
          # apt.postgresql.org are release builds where Assert() compiles to a no-op,
          # and there is no runtime toggle or debug apt package — cassert is compile-time
          # only. This run is the only way we get Postgres assertion coverage in CI.
          - pg_version: 18
            pg_impl: pgrx
            cpu: 8
            ram: 16
            runner_family: c7g+c8g
            runner_image: ubuntu24-full-arm64
            arch: arm64
          # Add AMD run for PG 18 (system Postgres)
          - pg_version: 18
            pg_impl: system
            cpu: 8
            ram: 16
            runner_family: c7a+c7i
            runner_image: ubuntu24-full-x64
            arch: amd64

    steps:
      - name: Checkout Git Repository
        uses: actions/checkout@v6

      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          cache: false
          rustflags: "" # Use .cargo/config.toml target-cpu flags

      - name: Extract pgrx Version
        id: pgrx
        run: echo "version=$(sed -nE 's/^pgrx = "=?([^"]+)"$/\1/p' Cargo.toml)" >> $GITHUB_OUTPUT

      # Cache notes:
      # - Do NOT add hashFiles('**/Cargo.lock') to shared-key — the action
      #   already hashes it internally. Putting it in shared-key defeats the
      #   restore-key fallback and causes cold builds on every lockfile change.
      # - Do NOT add arch to shared-key — the action includes the Rust target
      #   triple in its envHash. Keeping shared-key arch-free lets all debug-mode
      #   workflows (lint, test, schemabot, etc.) share the same cache.
      # PR branches restore from the default branch cache but don't save their own
      # entries — this prevents cache thrashing that fills the 30 GB limit and causes
      # every entry to be evicted before reuse. The nightly schedule run warms caches
      # for all PG versions on the default branch.
      - name: Install Rust Cache
        uses: swatinem/rust-cache@v2
        with:
          prefix-key: "v2-rust-cache"
          shared-key: pg${{ matrix.pg_version }}
          cache-targets: true
          cache-all-crates: true
          save-if: ${{ github.event_name == 'schedule' && github.ref_name == github.event.repository.default_branch }}

      - name: Install required system tools
        run: |
          sudo apt-get update && sudo apt-get install -y --no-install-recommends \
            lsof build-essential clang flex bison pkg-config ca-certificates git \
            libreadline-dev zlib1g-dev libssl-dev libkrb5-dev liblz4-dev libzstd-dev \
            libcurl4-openssl-dev

      - name: Install llvm-tools-preview
        run: rustup component add llvm-tools-preview

      - name: Install pgrx
        run: cargo install -j $(nproc) --locked cargo-pgrx --version "${{ steps.pgrx.outputs.version }}" --debug

      - name: Install cargo-llvm-cov
        run: command -v cargo-llvm-cov || cargo install -j $(nproc) cargo-llvm-cov --locked --debug

      # ---------- System-managed Postgres setup ----------
      - name: Install & Configure Supported PostgreSQL Version (system)
        if: matrix.pg_impl == 'system'
        run: |
          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
          sudo apt-get update && sudo apt-get install -y postgresql-${{ matrix.pg_version }} postgresql-server-dev-${{ matrix.pg_version }}
          echo "/usr/lib/postgresql/${{ matrix.pg_version }}/bin" >> $GITHUB_PATH

      - name: Initialize pgrx environment (system)
        if: matrix.pg_impl == 'system'
        run: cargo pgrx init "--pg${{ matrix.pg_version }}=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config"

      # ---------- pgrx-managed Postgres setup ----------
      - name: Cache pgrx-compiled Postgres
        if: matrix.pg_impl == 'pgrx'
        id: cache-pgrx-pg
        uses: actions/cache@v5
        with:
          path: ~/.pgrx
          key: pgrx-pg-${{ matrix.pg_version }}-${{ steps.pgrx.outputs.version }}-${{ runner.os }}-${{ runner.arch }}

      - name: Initialize pgrx environment (pgrx download)
        if: matrix.pg_impl == 'pgrx' && steps.cache-pgrx-pg.outputs.cache-hit != 'true'
        run: cargo pgrx init "--pg${{ matrix.pg_version }}=download" --configure-flag="--without-readline"

      # Needed for hybrid search unit/integration tests
      - name: Install pgvector (system)
        if: matrix.pg_impl == 'system'
        run: |
          git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
          cd pgvector/
          sudo PG_CONFIG=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config make -j
          sudo PG_CONFIG=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config make install -j

      - name: Install pgvector (pgrx)
        if: matrix.pg_impl == 'pgrx'
        run: |
          git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
          cd pgvector/
          PG_CONFIG=~/.pgrx/${{ matrix.pg_version }}.*/pgrx-install/bin/pg_config make -j
          PG_CONFIG=~/.pgrx/${{ matrix.pg_version }}.*/pgrx-install/bin/pg_config make install -j

      # Needed for Citus compatibility tests
      # Citus 13 supports PG 15-17, Citus 14 supports PG 16-18
      - name: Build & install Citus 13 (PG 15)
        if: matrix.pg_version == 15
        env:
          CITUS_VERSION: v13.0.1
        run: |
          set -euxo pipefail

          git clone --depth 1 --branch "${CITUS_VERSION}" https://github.com/citusdata/citus.git
          cd citus

          if [ "${{ matrix.pg_impl }}" = "system" ]; then
            PG_CONFIG="/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config"
          else
            PG_CONFIG="$(ls -d ~/.pgrx/${{ matrix.pg_version }}.*/pgrx-install/bin | head -n1)/pg_config"
          fi

          PG_CONFIG="${PG_CONFIG}" ./configure --without-libcurl
          PG_CONFIG="${PG_CONFIG}" make -j"$(nproc)"
          sudo PG_CONFIG="${PG_CONFIG}" make install -j"$(nproc)"

      - name: Build & install Citus 14 (PG 16-18)
        if: matrix.pg_version >= 16
        env:
          CITUS_VERSION: v14.0.0
        run: |
          set -euxo pipefail

          git clone --depth 1 --branch "${CITUS_VERSION}" https://github.com/citusdata/citus.git
          cd citus

          if [ "${{ matrix.pg_impl }}" = "system" ]; then
            PG_CONFIG="/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config"
          else
            PG_CONFIG="$(ls -d ~/.pgrx/${{ matrix.pg_version }}.*/pgrx-install/bin | head -n1)/pg_config"
          fi

          PG_CONFIG="${PG_CONFIG}" ./configure
          PG_CONFIG="${PG_CONFIG}" make -j"$(nproc)"
          sudo PG_CONFIG="${PG_CONFIG}" make install -j"$(nproc)"

      - name: Add extensions to shared_preload_libraries
        working-directory: /home/runner/.pgrx/data-${{ matrix.pg_version }}/
        run: |
          sed -i "s/^#shared_preload_libraries = .*/shared_preload_libraries = 'citus,pg_search'/" postgresql.conf
          sed -i "s/^max_connections = .*/max_connections = 300/" postgresql.conf
          echo "citus.node_conninfo = 'sslmode=prefer'" >> postgresql.conf

      - name: Compile & install pg_search extension (system)
        if: matrix.pg_impl == 'system'
        working-directory: pg_search/
        run: cargo pgrx install --sudo --features block_tracker --pg-config="/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config"

      # Coverage instrumentation runs on schedule + push-to-main, plus PG 18
      # entries on PRs (so Codecov has fresh data per PR). Older-version PR
      # entries run plain cargo test for ~30-40% wall-clock reduction.
      - name: Source LLVM coverage environment (pgrx)
        if: matrix.pg_impl == 'pgrx' && (github.event_name != 'pull_request' || matrix.pg_version == 18)
        run: |
          source <(cargo llvm-cov show-env --export-prefix)
          cargo llvm-cov clean --workspace
          env | grep -E '^(RUSTC_WRAPPER=|LLVM_PROFILE_FILE=|CARGO_LLVM_COV|__CARGO_LLVM_COV)' >> "$GITHUB_ENV"

      - name: Compile & install pg_search extension (pgrx)
        if: matrix.pg_impl == 'pgrx'
        working-directory: pg_search/
        run: cargo pgrx install --pg-config ~/.pgrx/${{ matrix.pg_version }}.*/pgrx-install/bin/pg_config --features="pg${{ matrix.pg_version }},block_tracker"

      # ---------- Start Postgres ----------
      - name: Start Postgres (system)
        if: matrix.pg_impl == 'system'
        working-directory: pg_search/
        run: |
          sudo chown -R $(whoami) /var/run/postgresql/
          RUST_BACKTRACE=1 cargo pgrx start pg${{ matrix.pg_version }}

      - name: Start Postgres and create database (pgrx)
        if: matrix.pg_impl == 'pgrx'
        working-directory: tests/
        run: |
          RUST_BACKTRACE=1 cargo pgrx start pg${{ matrix.pg_version }}
          # The cached ~/.pgrx directory includes the data directory, so restore hits
          # can already contain a pg_search database from a previous run.
          ~/.pgrx/${{ matrix.pg_version }}.*/pgrx-install/bin/dropdb --if-exists -p 288${{ matrix.pg_version }} -h localhost pg_search
          ~/.pgrx/${{ matrix.pg_version }}.*/pgrx-install/bin/createdb -p 288${{ matrix.pg_version }} -h localhost pg_search

      # ---------- Tests ----------
      - name: Run pg_search Integration Tests (system)
        if: matrix.pg_impl == 'system'
        run: |
          export DATABASE_URL=postgresql://localhost:288${{ matrix.pg_version }}/postgres
          export PG_CONFIG=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config
          if [[ "${{ github.event_name }}" == "pull_request" && "${{ matrix.pg_version }}" != "18" ]]; then
            RUST_BACKTRACE=1 cargo test --jobs $(nproc) --package tests --package tokenizers
          else
            RUST_BACKTRACE=1 cargo llvm-cov --jobs $(nproc) --package tests --package tokenizers --lcov --output-path coverage-integration-${{ matrix.pg_version }}-${{ matrix.pg_impl }}-${{ matrix.arch }}.lcov
          fi

      - name: Run pg_search Unit Tests (system)
        if: matrix.pg_impl == 'system'
        working-directory: pg_search/
        run: |
          sudo chown -R $(whoami) /usr/share/postgresql/${{ matrix.pg_version }}/ /usr/lib/postgresql/${{ matrix.pg_version }}/
          export DATABASE_URL=postgresql://localhost:288${{ matrix.pg_version }}/postgres
          if [[ "${{ github.event_name }}" == "pull_request" && "${{ matrix.pg_version }}" != "18" ]]; then
            RUST_BACKTRACE=1 cargo test --jobs $(nproc) --features pg${{ matrix.pg_version }} --no-default-features
          else
            RUST_BACKTRACE=1 cargo llvm-cov --jobs $(nproc) --features pg${{ matrix.pg_version }} --no-default-features --lcov --output-path ../coverage-unit-${{ matrix.pg_version }}-${{ matrix.pg_impl }}-${{ matrix.arch }}.lcov
          fi

      - name: Run pg_search Integration Tests (pgrx-managed)
        if: matrix.pg_impl == 'pgrx'
        run: RUST_BACKTRACE=1 DATABASE_URL=postgresql://localhost:288${{ matrix.pg_version }}/pg_search cargo test --jobs $(nproc) --no-default-features --package tests --package tokenizers -- --skip replication --skip ephemeral

      - name: Run pg_search Regression Tests (pgrx-managed)
        if: matrix.pg_impl == 'pgrx'
        run: |
          set -e
          if ! cargo pgrx regress --package pg_search pg${{ matrix.pg_version }} --auto; then
            git diff
            exit 1
          fi

      - name: Stop PostgreSQL to flush coverage data (pgrx)
        if: matrix.pg_impl == 'pgrx' && always()
        working-directory: pg_search/
        run: cargo pgrx stop pg${{ matrix.pg_version }}

      - name: Generate combined LCOV report (pgrx)
        if: matrix.pg_impl == 'pgrx' && (github.event_name != 'pull_request' || matrix.pg_version == 18) && always()
        run: cargo llvm-cov report --lcov --output-path coverage-combined-pgrx-${{ matrix.pg_version }}.lcov

      # runs-on's extras=s3-cache proxies ACTIONS_RESULTS_URL to its local cache
      # server (for s3-backed actions/cache), but actions/upload-artifact@v4+ uses
      # the same env var and ends up POSTing artifacts to the cache proxy. Pin the
      # env vars back to GitHub's canonical artifact endpoint just for this step
      # so the upload bypasses the proxy.
      - name: Upload Coverage Artifact
        if: (github.event_name != 'pull_request' || matrix.pg_version == 18) && always()
        env:
          ACTIONS_RESULTS_URL: https://results-receiver.actions.githubusercontent.com/
          ACTIONS_CACHE_SERVICE_V2: ""
        uses: actions/upload-artifact@v7
        with:
          name: coverage-${{ matrix.pg_version }}-${{ matrix.pg_impl }}-${{ matrix.arch }}
          path: coverage-*.lcov
          retention-days: 1

      - name: Print the Postgres Logs
        if: always()
        run: cat ~/.pgrx/${{ matrix.pg_version}}.log

  upload-coverage:
    name: Upload Combined Coverage to Codecov
    runs-on: ubuntu-latest
    needs: test-pg_search
    if: success()
    steps:
      - name: Checkout Git Repository
        uses: actions/checkout@v6

      - name: Download All Coverage Artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: coverage-*
          path: coverage-reports
          merge-multiple: true

      - name: Upload to Codecov
        uses: codecov/codecov-action@v6
        with:
          directory: ./coverage-reports
          files: "*.lcov"
          fail_ci_if_error: false
          verbose: true
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}