test-pg_search-antithesis.yml

# workflows/test-pg_search-antithesis.yml
#
# Test pg_search (Antithesis)
# Build the current commits of pg_search and the selected driver image(s) as
# Docker images, push them to Antithesis, and trigger one Antithesis test run
# per selected workload (stressgres and/or proptests).

name: Test pg_search (Antithesis)

on:
  # Scheduled runs are only executed on paradedb-enterprise; on paradedb/paradedb
  # (community) the job-level `if` below short-circuits the schedule trigger. The cron
  # entry stays here so the same workflow file can be synced between the two repos.
  schedule:
    - cron: "0 2 * * 1,4" # At 02:00 UTC on Monday and Thursday (Sunday night and Wednesday night)
  pull_request:
    types: [labeled]
  workflow_dispatch:
    inputs:
      workload:
        type: choice
        required: false
        default: all
        description: Which workload(s) to run on Antithesis
        options:
          - all
          - stressgres
          - proptests
      duration:
        type: string
        required: false
        description: Test duration (minutes, default 360, i.e., 6 hours)
      notebook:
        type: string
        required: false
        description: Antithesis notebook name (default 'paradedb')
      emails:
        type: string
        required: false
        description: Email recipients (default notifies developers@paradedb.com)

# No concurrency group since we want to allow multiple test runs to run simultaneously

env:
  PG_VERSION: "18"

jobs:
  # Resolve which workload(s) to run based on the trigger:
  # - schedule: all
  # - workflow_dispatch: from `inputs.workload` (default all)
  # - pull_request labeled: `antithesis` => all, `antithesis-stressgres` => stressgres,
  #   `antithesis-proptests` => proptests, anything else => skip
  #
  # For pull_request triggers, also removes the triggering label up front so it
  # can be re-applied to re-run.
  setup:
    name: Resolve Workloads
    runs-on: ubuntu-latest
    if: |
      (github.event_name == 'schedule' && github.repository == 'paradedb/paradedb-enterprise')
      || github.event_name == 'workflow_dispatch'
      || (github.event_name == 'pull_request' && contains(fromJSON('["antithesis","antithesis-stressgres","antithesis-proptests"]'), github.event.label.name))
    permissions:
      pull-requests: write
    outputs:
      workloads: ${{ steps.compute.outputs.workloads }}
    steps:
      - name: Remove Trigger Label
        if: github.event_name == 'pull_request'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          gh pr edit ${{ github.event.pull_request.number }} \
            --repo ${{ github.repository }} \
            --remove-label "${{ github.event.label.name }}"

      - id: compute
        env:
          EVENT: ${{ github.event_name }}
          INPUT_WORKLOAD: ${{ github.event.inputs.workload }}
          LABEL: ${{ github.event.label.name }}
        run: |
          set -euo pipefail
          WORKLOADS='[]'
          case "$EVENT" in
            schedule)
              WORKLOADS='["stressgres","proptests"]'
              ;;
            workflow_dispatch)
              case "${INPUT_WORKLOAD:-all}" in
                stressgres) WORKLOADS='["stressgres"]' ;;
                proptests)  WORKLOADS='["proptests"]'  ;;
                all|"")     WORKLOADS='["stressgres","proptests"]' ;;
              esac
              ;;
            pull_request)
              case "$LABEL" in
                antithesis)            WORKLOADS='["stressgres","proptests"]' ;;
                antithesis-stressgres) WORKLOADS='["stressgres"]' ;;
                antithesis-proptests)  WORKLOADS='["proptests"]'  ;;
              esac
              ;;
          esac
          echo "workloads=$WORKLOADS" >> "$GITHUB_OUTPUT"
          echo "Resolved workloads: $WORKLOADS"

  build-antithesis-pg_search-deb:
    name: Build Antithesis pg_search .deb
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=m7a+m7i
      - cpu=16
      - ram=64
      - image=ubuntu24-full-x64
      - extras=s3-cache
      - volume=80gb # default runner ships with 30GB — see https://runs-on.com/runners/linux/
    container:
      image: debian:13-slim
    if: |
      (github.event_name == 'schedule' && github.repository == 'paradedb/paradedb-enterprise')
      || github.event_name == 'workflow_dispatch'
      || (github.event_name == 'pull_request' && contains(fromJSON('["antithesis","antithesis-stressgres","antithesis-proptests"]'), github.event.label.name))
    env:
      COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}

    steps:
      - name: Install Dependencies
        run: DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y sudo wget git ca-certificates curl gnupg gpg lsb-release pkg-config libssl-dev

      - name: Checkout Git Repository
        uses: actions/checkout@v7
        with:
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}

      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          cache: false # Disable cache on publish workflows
          rustflags: "" # Use .cargo/config.toml target-cpu flags

      - name: Retrieve OS Version
        id: version
        run: |
          OS_VERSION="$(lsb_release -cs)"
          echo "OS Version: $OS_VERSION"
          echo "os_version=$OS_VERSION" >> $GITHUB_OUTPUT

      - name: Install & Configure Supported PostgreSQL Version
        run: |
          # Create keyrings dir (works on Debian/Ubuntu)
          sudo install -d -m 0755 /usr/share/keyrings

          # Download PostgreSQL signing key & add PostgreSQL APT repository with signed-by
          wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /usr/share/keyrings/postgresql.asc > /dev/null
          sudo sh -c 'echo "deb [signed-by=/usr/share/keyrings/postgresql.asc] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'

          # Install PostgreSQL + build tooling
          DEBIAN_FRONTEND=noninteractive sudo apt-get update
          DEBIAN_FRONTEND=noninteractive sudo apt-get install -y postgresql-${{ env.PG_VERSION }} postgresql-server-dev-${{ env.PG_VERSION }} debhelper devscripts dput gnupg binutils

          # Add Postgres bin dir to PATH for subsequent steps
          echo "/usr/lib/postgresql/${{ env.PG_VERSION }}/bin" >> "$GITHUB_PATH"

      - name: Extract pgrx Version
        id: pgrx
        run: echo version=$(sed -nE 's/^pgrx = "=?([^"]+)"$/\1/p' Cargo.toml) >> $GITHUB_OUTPUT

      - name: Cache pgrx binary
        id: cache-pgrx
        uses: actions/cache@v6
        with:
          path: ~/.cargo/bin/cargo-pgrx
          key: cargo-pgrx-${{ steps.pgrx.outputs.version }}-${{ runner.os }}-${{ runner.arch }}

      - name: Install pgrx
        if: steps.cache-pgrx.outputs.cache-hit != 'true'
        run: cargo install --locked cargo-pgrx --version ${{ steps.pgrx.outputs.version }} --debug

      - name: Initialize pgrx environment
        working-directory: pg_search/
        run: |
          PG_CONFIG_PATH="/usr/lib/postgresql/${{ env.PG_VERSION }}/bin/pg_config"
          cargo pgrx init --pg${{ env.PG_VERSION }}=$PG_CONFIG_PATH

      # The Antithesis build adds a sancov pass (-Cpasses=sancov-module) for coverage instrumentation.
      # ThinLTO drops Rust symbols under that pass, producing a pg_search.so that fails to dlopen, so disable LTO here.
      - name: Package pg_search Extension with pgrx
        working-directory: pg_search/
        run: |
          sudo curl -fsSL -o /usr/lib/libvoidstar.so https://antithesis.com/assets/instrumentation/libvoidstar.so
          PG_CONFIG_PATH="/usr/lib/postgresql/${{ env.PG_VERSION }}/bin/pg_config"
          RUSTFLAGS="-Ccodegen-units=1 -Cpasses=sancov-module -Cllvm-args=-sanitizer-coverage-level=3 -Cllvm-args=-sanitizer-coverage-trace-pc-guard -Clink-args=-Wl,--build-id -lvoidstar" \
          CARGO_PROFILE_RELEASE_WITH_DEBUG_LTO=off \
            cargo pgrx package --profile=release-with-debug --pg-config "$PG_CONFIG_PATH"

      # Catch the broken-symbol case above before shipping to Antithesis.
      # Undefined Rust symbols (_ZN/_R) mean a broken .so; Postgres C symbols stay undefined and are fine.
      - name: Verify pg_search Has No Unresolved Rust Symbols
        run: |
          LIB_PATH="target/release-with-debug/pg_search-pg${{ env.PG_VERSION }}/usr/lib/postgresql/${{ env.PG_VERSION }}/lib/pg_search.so"
          undefined_rust="$(nm -D -u "$LIB_PATH" | awk '{print $NF}' | grep -E '^_(ZN|R)' || true)"
          if [ -n "$undefined_rust" ]; then
            echo "::error::pg_search.so has unresolved Rust symbols — the build is broken and would fail to load:"
            echo "$undefined_rust"
            exit 1
          fi
          echo "OK: pg_search.so has no unresolved Rust symbols"

      - name: Split pg_search Debug Symbols
        run: |
          LIB_PATH="target/release-with-debug/pg_search-pg${{ env.PG_VERSION }}/usr/lib/postgresql/${{ env.PG_VERSION }}/lib/pg_search.so"
          objcopy --only-keep-debug "${LIB_PATH}" "${LIB_PATH}.dbg"
          objcopy --strip-debug "${LIB_PATH}"
          objcopy --add-gnu-debuglink="${LIB_PATH}.dbg" "${LIB_PATH}"

      - name: Create .deb Package
        run: |
          # Create installable package
          mkdir archive
          cp `find target/release-with-debug -type f -name "pg_search*"` archive
          package_dir=pg_search-antithesis-${{ steps.version.outputs.os_version }}-amd64-pg${{ env.PG_VERSION }}-${{ env.COMMIT_SHA }}

          mkdir -p ${package_dir}/usr/lib/postgresql/${{ env.PG_VERSION }}/lib
          mkdir -p ${package_dir}/usr/share/postgresql/${{ env.PG_VERSION }}/extension
          cp archive/*.so* ${package_dir}/usr/lib/postgresql/${{ env.PG_VERSION }}/lib
          cp archive/*.control ${package_dir}/usr/share/postgresql/${{ env.PG_VERSION }}/extension
          cp archive/*.sql ${package_dir}/usr/share/postgresql/${{ env.PG_VERSION }}/extension

          # Create control file (package name cannot have underscore)
          mkdir -p ${package_dir}/DEBIAN
          CONTROL_FILE="${package_dir}/DEBIAN/control"
          echo 'Package: postgresql-${{ env.PG_VERSION }}-pg-search' >> $CONTROL_FILE
          echo 'Version: 0.0.0' >> $CONTROL_FILE
          echo 'Section: database' >> $CONTROL_FILE
          echo 'Priority: optional' >> $CONTROL_FILE
          echo 'Architecture: amd64' >> $CONTROL_FILE
          echo 'Depends: postgresql-${{ env.PG_VERSION }}' >> $CONTROL_FILE
          echo 'Maintainer: ParadeDB <support@paradedb.com>' >> $CONTROL_FILE
          echo 'Description: Postgres for Search and Analytics' >> $CONTROL_FILE

          # Create .deb package
          sudo chown -R root:root ${package_dir}
          sudo chmod -R 755 ${package_dir}
          sudo dpkg-deb -Zxz --build --root-owner-group ${package_dir}

      - name: Upload pg_search .deb
        env:
          ACTIONS_RESULTS_URL: https://results-receiver.actions.githubusercontent.com/
          ACTIONS_CACHE_SERVICE_V2: ""
        uses: actions/upload-artifact@v7
        with:
          name: pg-search-antithesis-deb-${{ env.COMMIT_SHA }}
          path: pg_search-antithesis-${{ steps.version.outputs.os_version }}-amd64-pg${{ env.PG_VERSION }}-${{ env.COMMIT_SHA }}.deb
          retention-days: 1

  antithesis-trigger-test-run:
    name: Test pg_search via Antithesis (${{ matrix.workload }})
    needs:
      - setup
      - build-antithesis-pg_search-deb
    if: needs.setup.outputs.workloads != '[]'
    # For pull_request events, github.sha is the synthetic refs/pull/<n>/merge SHA
    # rather than the PR head — use the head SHA so image tags and the run
    # description match commits visible in the PR's git history.
    env:
      COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
    runs-on:
      # General-purpose m-family — Docker buildx is RAM-heavy.
      # See https://runs-on.com/configuration/job-labels/ — runs in us-east-1.
      - runs-on=${{ github.run_id }}
      - family=m7a+m7i
      - cpu=16
      - ram=64
      - image=ubuntu24-full-x64
      - volume=80gb # default runner ships with 30GB — see https://runs-on.com/runners/linux/
      - extras=s3-cache
    strategy:
      fail-fast: false
      matrix:
        workload: ${{ fromJSON(needs.setup.outputs.workloads) }}

    steps:
      - name: Checkout Git Repository
        uses: actions/checkout@v7
        with:
          ref: ${{ env.COMMIT_SHA }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4

      - name: Download Antithesis pg_search .deb
        env:
          ACTIONS_RESULTS_URL: https://results-receiver.actions.githubusercontent.com/
          ACTIONS_CACHE_SERVICE_V2: ""
        uses: actions/download-artifact@v8
        with:
          name: pg-search-antithesis-deb-${{ env.COMMIT_SHA }}
          path: docker

      - name: Log Into Antithesis Registry
        uses: docker/login-action@v4
        with:
          registry: us-central1-docker.pkg.dev
          username: _json_key
          password: ${{ secrets.ANTITHESIS_REGISTRY_KEY }}

      # The pg_version-tag Docker tag syntax is necessary for our CloudNativePG Helm chart
      - name: Extract Workload Metadata
        id: extract-workload-metadata
        uses: docker/metadata-action@v6
        with:
          images: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/paradedb
          tags: |
            type=raw,value=${{ env.PG_VERSION }}-${{ env.COMMIT_SHA }}
            type=raw,value=antithesis-latest

      # Antithesis only supports x86_64 instruction set
      # https://antithesis.com/docs/getting_started/setup/?sid=6534.37&sterm=x86&marks=x86#containerize-your-software
      - name: Build and Push Workload Docker Image to Antithesis Registry
        uses: docker/build-push-action@v7
        with:
          context: docker
          platforms: linux/amd64
          file: docker/Dockerfile.antithesis-${{ env.PG_VERSION }}
          push: true
          tags: ${{ steps.extract-workload-metadata.outputs.tags }}
          labels: ${{ steps.extract-workload-metadata.outputs.labels }}
          build-args: |
            COMMIT_SHA=${{ env.COMMIT_SHA }}

      - name: Patch ParadeDB K8s Manifest With Workload Image Tagged With Git SHA
        working-directory: docker/manifests/
        run: |
          sed -i "s|imageName: .*|imageName: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/paradedb:${{ env.PG_VERSION }}-${{ env.COMMIT_SHA }}|" antithesis-paradedb.yaml
          cat antithesis-paradedb.yaml

      - name: Patch Driver K8s Manifest With Driver Image Tagged With Git SHA
        working-directory: docker/manifests/
        run: |
          sed -i "s|image: .*|image: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/${{ matrix.workload }}:${{ env.COMMIT_SHA }}|" antithesis-${{ matrix.workload }}.yaml
          cat antithesis-${{ matrix.workload }}.yaml

      - name: Extract Config Metadata
        id: extract-config-metadata
        uses: docker/metadata-action@v6
        with:
          images: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/paradedb-config-${{ matrix.workload }}
          tags: |
            type=raw,value=${{ env.COMMIT_SHA }}
            type=raw,value=antithesis-latest

      - name: Build and Push Config Docker Image to Antithesis Registry
        uses: docker/build-push-action@v7
        with:
          context: .
          platforms: linux/amd64
          file: docker/Dockerfile.config.${{ matrix.workload }}
          push: true
          tags: ${{ steps.extract-config-metadata.outputs.tags }}
          labels: ${{ steps.extract-config-metadata.outputs.labels }}

      - name: Extract Driver Metadata
        id: extract-driver-metadata
        uses: docker/metadata-action@v6
        with:
          images: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/${{ matrix.workload }}
          tags: |
            type=raw,value=${{ env.COMMIT_SHA }}
            type=raw,value=antithesis-latest

      # Antithesis only supports x86_64 instruction set
      # https://antithesis.com/docs/getting_started/setup/?sid=6534.37&sterm=x86&marks=x86#containerize-your-software
      - name: Build and Push Driver Docker Image to Antithesis Registry
        uses: docker/build-push-action@v7
        with:
          context: .
          platforms: linux/amd64
          file: docker/Dockerfile.${{ matrix.workload }}
          push: true
          tags: ${{ steps.extract-driver-metadata.outputs.tags }}
          labels: ${{ steps.extract-driver-metadata.outputs.labels }}

      # antithesis.duration is in minutes, 360 minutes = 6 hours
      # antithesis.is_ephemeral ensures runs triggered from PRs don't impact the history
      # antithesis.source ensures Community and Enteprise have separate histories
      - name: Trigger Antithesis Test
        uses: antithesishq/antithesis-trigger-action@v0.12
        with:
          notebook_name: ${{ github.event.inputs.notebook || 'paradedb' }}
          tenant: paradedb
          username: ${{ vars.ANTITHESIS_USERNAME }}
          password: ${{ secrets.ANTITHESIS_PASSWORD }}
          github_token: ${{ secrets.ANTITHESIS_GITHUB_TOKEN }}
          config_image: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/paradedb-config-${{ matrix.workload }}:${{ env.COMMIT_SHA }}
          images: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/paradedb:${{ env.PG_VERSION }}-${{ env.COMMIT_SHA }}
          description: "[${{ github.repository }}] ${{ matrix.workload }} CI for ${{ github.ref_name }} (commit ${{ env.COMMIT_SHA }})"
          email_recipients: ${{ github.event.inputs.emails || 'developers@paradedb.com' }}
          additional_parameters: |-
            antithesis.duration=${{ github.event.inputs.duration || (github.event_name == 'pull_request' && '60') || '360' }}
            antithesis.is_ephemeral=${{ github.event_name == 'pull_request' }}
            antithesis.source=paradedb
            custom.network_fault_exclusion_patterns=coredns,${{ matrix.workload }}
            custom.container_faults_stop_exclusion_patterns=coredns,${{ matrix.workload }},paradedb,cloudnative-pg,logical-replication-publisher
            custom.container_faults_kill_exclusion_patterns=coredns,${{ matrix.workload }},paradedb,cloudnative-pg,logical-replication-publisher

  # Ping Slack if any of the nightly (scheduled) jobs failed. Scheduled runs only execute on
  # paradedb-enterprise (see the `if` on `setup`), so this notification only fires there.
  notify-slack-on-failure:
    name: Notify Slack on Failure
    needs: [setup, build-antithesis-pg_search-deb, antithesis-trigger-test-run]
    if: |
      always()
      && github.event_name == 'schedule'
      && (needs.setup.result == 'failure'
          || needs.build-antithesis-pg_search-deb.result == 'failure'
          || needs.antithesis-trigger-test-run.result == 'failure')
    runs-on: ubuntu-latest
    env:
      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GITHUB_CHANNEL_WEBHOOK_URL }}
    steps:
      - name: Determine failed jobs
        id: failed-jobs
        run: |
          FAILED_JOBS=""
          if [ "${{ needs.setup.result }}" == "failure" ]; then
            FAILED_JOBS="Resolve Workloads, $FAILED_JOBS"
          fi
          if [ "${{ needs.build-antithesis-pg_search-deb.result }}" == "failure" ]; then
            FAILED_JOBS="Build Antithesis pg_search .deb, $FAILED_JOBS"
          fi
          if [ "${{ needs.antithesis-trigger-test-run.result }}" == "failure" ]; then
            FAILED_JOBS="Test pg_search via Antithesis, $FAILED_JOBS"
          fi
          FAILED_JOBS="${FAILED_JOBS%, }"
          echo "failed_jobs=$FAILED_JOBS" >> "$GITHUB_OUTPUT"

      - name: Send Slack notification
        run: |
          if [ -n "$SLACK_WEBHOOK_URL" ]; then
            JSON_DATA="{
              \"text\": \"🚨 Nightly Antithesis Run Failed - <!here>\",
              \"attachments\": [
                {
                  \"color\": \"danger\",
                  \"fields\": [
                    {\"title\": \"Repository\", \"value\": \"${{ github.repository }}\", \"short\": true},
                    {\"title\": \"Workflow\", \"value\": \"${{ github.workflow }}\", \"short\": true},
                    {\"title\": \"Run ID\", \"value\": \"${{ github.run_id }}\", \"short\": true},
                    {\"title\": \"View Logs\", \"value\": \"<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Click here>\", \"short\": true},
                    {\"title\": \"Failed Jobs\", \"value\": \"${{ steps.failed-jobs.outputs.failed_jobs }}\", \"short\": false}
                  ]
                }
              ]
            }"

            curl -X POST -H 'Content-type: application/json' \
              --data "$JSON_DATA" \
              "$SLACK_WEBHOOK_URL"
          else
            echo "Slack webhook not configured, skipping notification"
          fi