Test pg_search (Antithesis) #14
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # workflows/test-pg_search-antithesis.yml | |
| # | |
| # Test pg_search (Antithesis) | |
| # Build the current commits of pg_search and the selected driver image(s) as | |
| # Docker images, push them to Antithesis, and trigger one Antithesis test run | |
| # per selected workload (stressgres and/or proptests). | |
| name: Test pg_search (Antithesis) | |
| on: | |
| # Scheduled runs are only executed on paradedb-enterprise; on paradedb/paradedb | |
| # (community) the job-level `if` below short-circuits the schedule trigger. The cron | |
| # entry stays here so the same workflow file can be synced between the two repos. | |
| schedule: | |
| - cron: "0 2 * * 1,4" # At 02:00 UTC on Monday and Thursday (Sunday night and Wednesday night) | |
| pull_request: | |
| types: [labeled] | |
| workflow_dispatch: | |
| inputs: | |
| workload: | |
| type: choice | |
| required: false | |
| default: all | |
| description: Which workload(s) to run on Antithesis | |
| options: | |
| - all | |
| - stressgres | |
| - proptests | |
| duration: | |
| type: string | |
| required: false | |
| description: Test duration (minutes, default 360, i.e., 6 hours) | |
| notebook: | |
| type: string | |
| required: false | |
| description: Antithesis notebook name (default 'paradedb') | |
| emails: | |
| type: string | |
| required: false | |
| description: Email recipients (default notifies developers@paradedb.com) | |
| # No concurrency group since we want to allow multiple test runs to run simultaneously | |
| env: | |
| PG_VERSION: "18" | |
| jobs: | |
| # Resolve which workload(s) to run based on the trigger: | |
| # - schedule: all | |
| # - workflow_dispatch: from `inputs.workload` (default all) | |
| # - pull_request labeled: `antithesis` => all, `antithesis-stressgres` => stressgres, | |
| # `antithesis-proptests` => proptests, anything else => skip | |
| # | |
| # For pull_request triggers, also removes the triggering label up front so it | |
| # can be re-applied to re-run. | |
| setup: | |
| name: Resolve Workloads | |
| runs-on: ubuntu-latest | |
| if: | | |
| (github.event_name == 'schedule' && github.repository == 'paradedb/paradedb-enterprise') | |
| || github.event_name == 'workflow_dispatch' | |
| || (github.event_name == 'pull_request' && contains(fromJSON('["antithesis","antithesis-stressgres","antithesis-proptests"]'), github.event.label.name)) | |
| permissions: | |
| pull-requests: write | |
| outputs: | |
| workloads: ${{ steps.compute.outputs.workloads }} | |
| steps: | |
| - name: Remove Trigger Label | |
| if: github.event_name == 'pull_request' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh pr edit ${{ github.event.pull_request.number }} \ | |
| --repo ${{ github.repository }} \ | |
| --remove-label "${{ github.event.label.name }}" | |
| - id: compute | |
| env: | |
| EVENT: ${{ github.event_name }} | |
| INPUT_WORKLOAD: ${{ github.event.inputs.workload }} | |
| LABEL: ${{ github.event.label.name }} | |
| run: | | |
| set -euo pipefail | |
| WORKLOADS='[]' | |
| case "$EVENT" in | |
| schedule) | |
| WORKLOADS='["stressgres","proptests"]' | |
| ;; | |
| workflow_dispatch) | |
| case "${INPUT_WORKLOAD:-all}" in | |
| stressgres) WORKLOADS='["stressgres"]' ;; | |
| proptests) WORKLOADS='["proptests"]' ;; | |
| all|"") WORKLOADS='["stressgres","proptests"]' ;; | |
| esac | |
| ;; | |
| pull_request) | |
| case "$LABEL" in | |
| antithesis) WORKLOADS='["stressgres","proptests"]' ;; | |
| antithesis-stressgres) WORKLOADS='["stressgres"]' ;; | |
| antithesis-proptests) WORKLOADS='["proptests"]' ;; | |
| esac | |
| ;; | |
| esac | |
| echo "workloads=$WORKLOADS" >> "$GITHUB_OUTPUT" | |
| echo "Resolved workloads: $WORKLOADS" | |
| build-antithesis-pg_search-deb: | |
| name: Build Antithesis pg_search .deb | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - family=m7a+m7i | |
| - cpu=16 | |
| - ram=64 | |
| - image=ubuntu24-full-x64 | |
| - extras=s3-cache | |
| - volume=80gb # default runner ships with 30GB — see https://runs-on.com/runners/linux/ | |
| container: | |
| image: debian:13-slim | |
| if: | | |
| (github.event_name == 'schedule' && github.repository == 'paradedb/paradedb-enterprise') | |
| || github.event_name == 'workflow_dispatch' | |
| || (github.event_name == 'pull_request' && contains(fromJSON('["antithesis","antithesis-stressgres","antithesis-proptests"]'), github.event.label.name)) | |
| env: | |
| COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} | |
| steps: | |
| - name: Install Dependencies | |
| run: DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y sudo wget git ca-certificates curl gnupg gpg lsb-release pkg-config libssl-dev | |
| - name: Checkout Git Repository | |
| uses: actions/checkout@v7 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| cache: false # Disable cache on publish workflows | |
| rustflags: "" # Use .cargo/config.toml target-cpu flags | |
| - name: Retrieve OS Version | |
| id: version | |
| run: | | |
| OS_VERSION="$(lsb_release -cs)" | |
| echo "OS Version: $OS_VERSION" | |
| echo "os_version=$OS_VERSION" >> $GITHUB_OUTPUT | |
| - name: Install & Configure Supported PostgreSQL Version | |
| run: | | |
| # Create keyrings dir (works on Debian/Ubuntu) | |
| sudo install -d -m 0755 /usr/share/keyrings | |
| # Download PostgreSQL signing key & add PostgreSQL APT repository with signed-by | |
| wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /usr/share/keyrings/postgresql.asc > /dev/null | |
| sudo sh -c 'echo "deb [signed-by=/usr/share/keyrings/postgresql.asc] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' | |
| # Install PostgreSQL + build tooling | |
| DEBIAN_FRONTEND=noninteractive sudo apt-get update | |
| DEBIAN_FRONTEND=noninteractive sudo apt-get install -y postgresql-${{ env.PG_VERSION }} postgresql-server-dev-${{ env.PG_VERSION }} debhelper devscripts dput gnupg binutils | |
| # Add Postgres bin dir to PATH for subsequent steps | |
| echo "/usr/lib/postgresql/${{ env.PG_VERSION }}/bin" >> "$GITHUB_PATH" | |
| - name: Extract pgrx Version | |
| id: pgrx | |
| run: echo version=$(sed -nE 's/^pgrx = "=?([^"]+)"$/\1/p' Cargo.toml) >> $GITHUB_OUTPUT | |
| - name: Cache pgrx binary | |
| id: cache-pgrx | |
| uses: actions/cache@v6 | |
| with: | |
| path: ~/.cargo/bin/cargo-pgrx | |
| key: cargo-pgrx-${{ steps.pgrx.outputs.version }}-${{ runner.os }}-${{ runner.arch }} | |
| - name: Install pgrx | |
| if: steps.cache-pgrx.outputs.cache-hit != 'true' | |
| run: cargo install --locked cargo-pgrx --version ${{ steps.pgrx.outputs.version }} --debug | |
| - name: Initialize pgrx environment | |
| working-directory: pg_search/ | |
| run: | | |
| PG_CONFIG_PATH="/usr/lib/postgresql/${{ env.PG_VERSION }}/bin/pg_config" | |
| cargo pgrx init --pg${{ env.PG_VERSION }}=$PG_CONFIG_PATH | |
| # The Antithesis build adds a sancov pass (-Cpasses=sancov-module) for coverage instrumentation. | |
| # ThinLTO drops Rust symbols under that pass, producing a pg_search.so that fails to dlopen, so disable LTO here. | |
| - name: Package pg_search Extension with pgrx | |
| working-directory: pg_search/ | |
| run: | | |
| sudo curl -fsSL -o /usr/lib/libvoidstar.so https://antithesis.com/assets/instrumentation/libvoidstar.so | |
| PG_CONFIG_PATH="/usr/lib/postgresql/${{ env.PG_VERSION }}/bin/pg_config" | |
| RUSTFLAGS="-Ccodegen-units=1 -Cpasses=sancov-module -Cllvm-args=-sanitizer-coverage-level=3 -Cllvm-args=-sanitizer-coverage-trace-pc-guard -Clink-args=-Wl,--build-id -lvoidstar" \ | |
| CARGO_PROFILE_RELEASE_WITH_DEBUG_LTO=off \ | |
| cargo pgrx package --profile=release-with-debug --pg-config "$PG_CONFIG_PATH" | |
| # Catch the broken-symbol case above before shipping to Antithesis. | |
| # Undefined Rust symbols (_ZN/_R) mean a broken .so; Postgres C symbols stay undefined and are fine. | |
| - name: Verify pg_search Has No Unresolved Rust Symbols | |
| run: | | |
| LIB_PATH="target/release-with-debug/pg_search-pg${{ env.PG_VERSION }}/usr/lib/postgresql/${{ env.PG_VERSION }}/lib/pg_search.so" | |
| undefined_rust="$(nm -D -u "$LIB_PATH" | awk '{print $NF}' | grep -E '^_(ZN|R)' || true)" | |
| if [ -n "$undefined_rust" ]; then | |
| echo "::error::pg_search.so has unresolved Rust symbols — the build is broken and would fail to load:" | |
| echo "$undefined_rust" | |
| exit 1 | |
| fi | |
| echo "OK: pg_search.so has no unresolved Rust symbols" | |
| - name: Split pg_search Debug Symbols | |
| run: | | |
| LIB_PATH="target/release-with-debug/pg_search-pg${{ env.PG_VERSION }}/usr/lib/postgresql/${{ env.PG_VERSION }}/lib/pg_search.so" | |
| objcopy --only-keep-debug "${LIB_PATH}" "${LIB_PATH}.dbg" | |
| objcopy --strip-debug "${LIB_PATH}" | |
| objcopy --add-gnu-debuglink="${LIB_PATH}.dbg" "${LIB_PATH}" | |
| - name: Create .deb Package | |
| run: | | |
| # Create installable package | |
| mkdir archive | |
| cp `find target/release-with-debug -type f -name "pg_search*"` archive | |
| package_dir=pg_search-antithesis-${{ steps.version.outputs.os_version }}-amd64-pg${{ env.PG_VERSION }}-${{ env.COMMIT_SHA }} | |
| mkdir -p ${package_dir}/usr/lib/postgresql/${{ env.PG_VERSION }}/lib | |
| mkdir -p ${package_dir}/usr/share/postgresql/${{ env.PG_VERSION }}/extension | |
| cp archive/*.so* ${package_dir}/usr/lib/postgresql/${{ env.PG_VERSION }}/lib | |
| cp archive/*.control ${package_dir}/usr/share/postgresql/${{ env.PG_VERSION }}/extension | |
| cp archive/*.sql ${package_dir}/usr/share/postgresql/${{ env.PG_VERSION }}/extension | |
| # Create control file (package name cannot have underscore) | |
| mkdir -p ${package_dir}/DEBIAN | |
| CONTROL_FILE="${package_dir}/DEBIAN/control" | |
| echo 'Package: postgresql-${{ env.PG_VERSION }}-pg-search' >> $CONTROL_FILE | |
| echo 'Version: 0.0.0' >> $CONTROL_FILE | |
| echo 'Section: database' >> $CONTROL_FILE | |
| echo 'Priority: optional' >> $CONTROL_FILE | |
| echo 'Architecture: amd64' >> $CONTROL_FILE | |
| echo 'Depends: postgresql-${{ env.PG_VERSION }}' >> $CONTROL_FILE | |
| echo 'Maintainer: ParadeDB <support@paradedb.com>' >> $CONTROL_FILE | |
| echo 'Description: Postgres for Search and Analytics' >> $CONTROL_FILE | |
| # Create .deb package | |
| sudo chown -R root:root ${package_dir} | |
| sudo chmod -R 755 ${package_dir} | |
| sudo dpkg-deb -Zxz --build --root-owner-group ${package_dir} | |
| - name: Upload pg_search .deb | |
| env: | |
| ACTIONS_RESULTS_URL: https://results-receiver.actions.githubusercontent.com/ | |
| ACTIONS_CACHE_SERVICE_V2: "" | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: pg-search-antithesis-deb-${{ env.COMMIT_SHA }} | |
| path: pg_search-antithesis-${{ steps.version.outputs.os_version }}-amd64-pg${{ env.PG_VERSION }}-${{ env.COMMIT_SHA }}.deb | |
| retention-days: 1 | |
| antithesis-trigger-test-run: | |
| name: Test pg_search via Antithesis (${{ matrix.workload }}) | |
| needs: | |
| - setup | |
| - build-antithesis-pg_search-deb | |
| if: needs.setup.outputs.workloads != '[]' | |
| # For pull_request events, github.sha is the synthetic refs/pull/<n>/merge SHA | |
| # rather than the PR head — use the head SHA so image tags and the run | |
| # description match commits visible in the PR's git history. | |
| env: | |
| COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} | |
| runs-on: | |
| # General-purpose m-family — Docker buildx is RAM-heavy. | |
| # See https://runs-on.com/configuration/job-labels/ — runs in us-east-1. | |
| - runs-on=${{ github.run_id }} | |
| - family=m7a+m7i | |
| - cpu=16 | |
| - ram=64 | |
| - image=ubuntu24-full-x64 | |
| - volume=80gb # default runner ships with 30GB — see https://runs-on.com/runners/linux/ | |
| - extras=s3-cache | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| workload: ${{ fromJSON(needs.setup.outputs.workloads) }} | |
| steps: | |
| - name: Checkout Git Repository | |
| uses: actions/checkout@v7 | |
| with: | |
| ref: ${{ env.COMMIT_SHA }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Download Antithesis pg_search .deb | |
| env: | |
| ACTIONS_RESULTS_URL: https://results-receiver.actions.githubusercontent.com/ | |
| ACTIONS_CACHE_SERVICE_V2: "" | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: pg-search-antithesis-deb-${{ env.COMMIT_SHA }} | |
| path: docker | |
| - name: Log Into Antithesis Registry | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: us-central1-docker.pkg.dev | |
| username: _json_key | |
| password: ${{ secrets.ANTITHESIS_REGISTRY_KEY }} | |
| # The pg_version-tag Docker tag syntax is necessary for our CloudNativePG Helm chart | |
| - name: Extract Workload Metadata | |
| id: extract-workload-metadata | |
| uses: docker/metadata-action@v6 | |
| with: | |
| images: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/paradedb | |
| tags: | | |
| type=raw,value=${{ env.PG_VERSION }}-${{ env.COMMIT_SHA }} | |
| type=raw,value=antithesis-latest | |
| # Antithesis only supports x86_64 instruction set | |
| # https://antithesis.com/docs/getting_started/setup/?sid=6534.37&sterm=x86&marks=x86#containerize-your-software | |
| - name: Build and Push Workload Docker Image to Antithesis Registry | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: docker | |
| platforms: linux/amd64 | |
| file: docker/Dockerfile.antithesis-${{ env.PG_VERSION }} | |
| push: true | |
| tags: ${{ steps.extract-workload-metadata.outputs.tags }} | |
| labels: ${{ steps.extract-workload-metadata.outputs.labels }} | |
| build-args: | | |
| COMMIT_SHA=${{ env.COMMIT_SHA }} | |
| - name: Patch ParadeDB K8s Manifest With Workload Image Tagged With Git SHA | |
| working-directory: docker/manifests/ | |
| run: | | |
| sed -i "s|imageName: .*|imageName: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/paradedb:${{ env.PG_VERSION }}-${{ env.COMMIT_SHA }}|" antithesis-paradedb.yaml | |
| cat antithesis-paradedb.yaml | |
| - name: Patch Driver K8s Manifest With Driver Image Tagged With Git SHA | |
| working-directory: docker/manifests/ | |
| run: | | |
| sed -i "s|image: .*|image: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/${{ matrix.workload }}:${{ env.COMMIT_SHA }}|" antithesis-${{ matrix.workload }}.yaml | |
| cat antithesis-${{ matrix.workload }}.yaml | |
| - name: Extract Config Metadata | |
| id: extract-config-metadata | |
| uses: docker/metadata-action@v6 | |
| with: | |
| images: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/paradedb-config-${{ matrix.workload }} | |
| tags: | | |
| type=raw,value=${{ env.COMMIT_SHA }} | |
| type=raw,value=antithesis-latest | |
| - name: Build and Push Config Docker Image to Antithesis Registry | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| platforms: linux/amd64 | |
| file: docker/Dockerfile.config.${{ matrix.workload }} | |
| push: true | |
| tags: ${{ steps.extract-config-metadata.outputs.tags }} | |
| labels: ${{ steps.extract-config-metadata.outputs.labels }} | |
| - name: Extract Driver Metadata | |
| id: extract-driver-metadata | |
| uses: docker/metadata-action@v6 | |
| with: | |
| images: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/${{ matrix.workload }} | |
| tags: | | |
| type=raw,value=${{ env.COMMIT_SHA }} | |
| type=raw,value=antithesis-latest | |
| # Antithesis only supports x86_64 instruction set | |
| # https://antithesis.com/docs/getting_started/setup/?sid=6534.37&sterm=x86&marks=x86#containerize-your-software | |
| - name: Build and Push Driver Docker Image to Antithesis Registry | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| platforms: linux/amd64 | |
| file: docker/Dockerfile.${{ matrix.workload }} | |
| push: true | |
| tags: ${{ steps.extract-driver-metadata.outputs.tags }} | |
| labels: ${{ steps.extract-driver-metadata.outputs.labels }} | |
| # antithesis.duration is in minutes, 360 minutes = 6 hours | |
| # antithesis.is_ephemeral ensures runs triggered from PRs don't impact the history | |
| # antithesis.source ensures Community and Enteprise have separate histories | |
| - name: Trigger Antithesis Test | |
| uses: antithesishq/antithesis-trigger-action@v0.12 | |
| with: | |
| notebook_name: ${{ github.event.inputs.notebook || 'paradedb' }} | |
| tenant: paradedb | |
| username: ${{ vars.ANTITHESIS_USERNAME }} | |
| password: ${{ secrets.ANTITHESIS_PASSWORD }} | |
| github_token: ${{ secrets.ANTITHESIS_GITHUB_TOKEN }} | |
| config_image: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/paradedb-config-${{ matrix.workload }}:${{ env.COMMIT_SHA }} | |
| images: us-central1-docker.pkg.dev/molten-verve-216720/paradedb-repository/paradedb:${{ env.PG_VERSION }}-${{ env.COMMIT_SHA }} | |
| description: "[${{ github.repository }}] ${{ matrix.workload }} CI for ${{ github.ref_name }} (commit ${{ env.COMMIT_SHA }})" | |
| email_recipients: ${{ github.event.inputs.emails || 'developers@paradedb.com' }} | |
| additional_parameters: |- | |
| antithesis.duration=${{ github.event.inputs.duration || (github.event_name == 'pull_request' && '60') || '360' }} | |
| antithesis.is_ephemeral=${{ github.event_name == 'pull_request' }} | |
| antithesis.source=paradedb | |
| custom.network_fault_exclusion_patterns=coredns,${{ matrix.workload }} | |
| custom.container_faults_stop_exclusion_patterns=coredns,${{ matrix.workload }},paradedb,cloudnative-pg,logical-replication-publisher | |
| custom.container_faults_kill_exclusion_patterns=coredns,${{ matrix.workload }},paradedb,cloudnative-pg,logical-replication-publisher | |
| # Ping Slack if any of the nightly (scheduled) jobs failed. Scheduled runs only execute on | |
| # paradedb-enterprise (see the `if` on `setup`), so this notification only fires there. | |
| notify-slack-on-failure: | |
| name: Notify Slack on Failure | |
| needs: [setup, build-antithesis-pg_search-deb, antithesis-trigger-test-run] | |
| if: | | |
| always() | |
| && github.event_name == 'schedule' | |
| && (needs.setup.result == 'failure' | |
| || needs.build-antithesis-pg_search-deb.result == 'failure' | |
| || needs.antithesis-trigger-test-run.result == 'failure') | |
| runs-on: ubuntu-latest | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GITHUB_CHANNEL_WEBHOOK_URL }} | |
| steps: | |
| - name: Determine failed jobs | |
| id: failed-jobs | |
| run: | | |
| FAILED_JOBS="" | |
| if [ "${{ needs.setup.result }}" == "failure" ]; then | |
| FAILED_JOBS="Resolve Workloads, $FAILED_JOBS" | |
| fi | |
| if [ "${{ needs.build-antithesis-pg_search-deb.result }}" == "failure" ]; then | |
| FAILED_JOBS="Build Antithesis pg_search .deb, $FAILED_JOBS" | |
| fi | |
| if [ "${{ needs.antithesis-trigger-test-run.result }}" == "failure" ]; then | |
| FAILED_JOBS="Test pg_search via Antithesis, $FAILED_JOBS" | |
| fi | |
| FAILED_JOBS="${FAILED_JOBS%, }" | |
| echo "failed_jobs=$FAILED_JOBS" >> "$GITHUB_OUTPUT" | |
| - name: Send Slack notification | |
| run: | | |
| if [ -n "$SLACK_WEBHOOK_URL" ]; then | |
| JSON_DATA="{ | |
| \"text\": \"🚨 Nightly Antithesis Run Failed - <!here>\", | |
| \"attachments\": [ | |
| { | |
| \"color\": \"danger\", | |
| \"fields\": [ | |
| {\"title\": \"Repository\", \"value\": \"${{ github.repository }}\", \"short\": true}, | |
| {\"title\": \"Workflow\", \"value\": \"${{ github.workflow }}\", \"short\": true}, | |
| {\"title\": \"Run ID\", \"value\": \"${{ github.run_id }}\", \"short\": true}, | |
| {\"title\": \"View Logs\", \"value\": \"<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Click here>\", \"short\": true}, | |
| {\"title\": \"Failed Jobs\", \"value\": \"${{ steps.failed-jobs.outputs.failed_jobs }}\", \"short\": false} | |
| ] | |
| } | |
| ] | |
| }" | |
| curl -X POST -H 'Content-type: application/json' \ | |
| --data "$JSON_DATA" \ | |
| "$SLACK_WEBHOOK_URL" | |
| else | |
| echo "Slack webhook not configured, skipping notification" | |
| fi |