Merge branch 'main' into copilot/add-for-clause-support

Author: joocer

.github/workflows/regression_suite.yaml

@@ -35,6 +35,12 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
+      - name: Install system packages (Ubuntu)
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo apt-get update
+          # We only need libssl-dev/pkg-config for libcrypto and build tooling
+          sudo apt-get install -y libssl-dev pkg-config
         
       - name: Install Requirements
         run: |
@@ -44,6 +50,20 @@ jobs:
           python -m uv pip install --upgrade -r $GITHUB_WORKSPACE/pyproject.toml
           python setup.py build_ext --inplace -j 4
 
+      - name: Print 'parquet' shared object dependencies
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          set -eux
+          SO_FILE=$(ls -1 opteryx/rugo/parquet*.so | head -n1)
+          echo "Located: $SO_FILE"
+          echo "File type: $(file $SO_FILE)"
+          echo "ldd output:"
+          ldd $SO_FILE || true
+          if ! ldd $SO_FILE | grep -q libcrypto; then
+            echo "ERROR: libcrypto not linked into $SO_FILE"
+            exit 1
+          fi
+
       - name: "Start Memcached Instance"
         run: |
             docker run -d --name my-memcached \
@@ -82,7 +102,6 @@ jobs:
           OPTERYX_DEBUG: 1
           MAX_LOCAL_BUFFER_CAPACITY: 100
           MAX_CACHE_EVICTIONS_PER_QUERY: 4
-          DATA_CATALOG_PROVIDER: 'ICEBERG'
           DATA_CATALOG_CONNECTION: '${{ secrets.DATA_CATALOG_CONNECTION }}'
           DATA_CATALOG_STORAGE: '${{ secrets.DATA_CATALOG_STORAGE }}'
           VALKEY_CONNECTION: '${{ secrets.VALKEY_CONNECTION }}'

.github/workflows/regression_suite_arm.yaml

@@ -26,6 +26,12 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           cache: pip
+      - name: Install system packages (Ubuntu ARM)
+        if: matrix.os == 'ubuntu-24.04-arm'
+        run: |
+          sudo apt-get update
+          # Only needed for libcrypto & build tools on ARM; zstd/snappy are vendored.
+          sudo apt-get install -y libssl-dev pkg-config
 
       - name: Install Rust
         uses: actions-rs/toolchain@v1
@@ -41,6 +47,23 @@ jobs:
           python -m uv pip install --upgrade -r $GITHUB_WORKSPACE/pyproject.toml
           python setup.py build_ext --inplace -j 4
 
+      - name: Print 'parquet' shared object dependencies
+        run: |
+          set -eux
+          # Attempt to locate the built parquet shared object and print its dynamic
+          # dependencies so we can verify that it was linked against libcrypto on Linux.
+          SO_FILE=$(ls -1 opteryx/rugo/parquet*.so | head -n1)
+          echo "Located: $SO_FILE"
+          echo "File type: $(file $SO_FILE)"
+          echo "ldd output:"
+          ldd $SO_FILE || true
+          if ! ldd $SO_FILE | grep -q libcrypto; then
+            echo "ERROR: libcrypto not linked into $SO_FILE"
+            exit 1
+          fi
+          # We vendor zstd & snappy into the parquet extension; they may not show
+          # up in ldd output as dynamic dependencies. Only assert libcrypto presence.
+
       - name: "Authenticate to Google Cloud"
         uses: google-github-actions/auth@v1
         with:
@@ -55,7 +78,4 @@ jobs:
           POSTGRES_USER: '${{ secrets.POSTGRES_USER }}'
           POSTGRES_PASSWORD: '${{ secrets.POSTGRES_PASSWORD }}'
           MEMCACHED_SERVER: 'localhost:11211'
-          DATA_CATALOG_PROVIDER: 'ICEBERG'
-          DATA_CATALOG_CONFIGURATION: '${{ secrets.DATA_CATALOG_CONFIGURATION }}'
-          EXPERIMENTAL_EXECUTION_ENGINE: 'true'
           RESOURCES_PATH: 'testdata'

.github/workflows/regression_suite_mac_ARM.yaml

@@ -41,14 +41,15 @@ jobs:
           python -m uv pip install --upgrade -r $GITHUB_WORKSPACE/tests/requirements.txt
           python -m uv pip install --upgrade -r $GITHUB_WORKSPACE/pyproject.toml
 
-      - name: Build on macOS universal2
+      - name: Build on macOS (arm64)
         shell: bash
         env:
           DEVELOPER_DIR: /Applications/Xcode.app/Contents/Developer
           MACOSX_DEPLOYMENT_TARGET: "10.15"
           ARCHFLAGS: -arch arm64
           PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }}
-        run: python setup.py build_ext --inplace -j 4
+        run: |
+          python setup.py build_ext --inplace -j 4
 
       - name: "Authenticate to Google Cloud"
         uses: google-github-actions/auth@v1
@@ -64,7 +65,5 @@ jobs:
           POSTGRES_USER: '${{ secrets.POSTGRES_USER }}'
           POSTGRES_PASSWORD: '${{ secrets.POSTGRES_PASSWORD }}'
           MEMCACHED_SERVER: 'localhost:11211'
-          DATA_CATALOG_PROVIDER: 'ICEBERG'
-          DATA_CATALOG_CONFIGURATION: '${{ secrets.DATA_CATALOG_CONFIGURATION }}'
           EXPERIMENTAL_EXECUTION_ENGINE: 'true'
           RESOURCES_PATH: 'testdata'

.github/workflows/regression_suite_python313t.yaml

@@ -114,7 +114,6 @@ jobs:
           OPTERYX_DEBUG: 1
           MAX_LOCAL_BUFFER_CAPACITY: 100
           MAX_CACHE_EVICTIONS_PER_QUERY: 4
-          DATA_CATALOG_PROVIDER: 'ICEBERG'
           DATA_CATALOG_CONNECTION: '${{ secrets.DATA_CATALOG_CONNECTION }}'
           DATA_CATALOG_STORAGE: '${{ secrets.DATA_CATALOG_STORAGE }}'
           VALKEY_CONNECTION: '${{ secrets.VALKEY_CONNECTION }}'

.github/workflows/release.yaml

@@ -90,7 +90,7 @@ jobs:
           toolchain: stable
           override: true
 
-      - name: Set up Python ${{ matrix.python-version }} x64
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
@@ -101,15 +101,19 @@ jobs:
       - name: Install Rust aarch64-apple-darwin target
         run: rustup target add aarch64-apple-darwin
 
-      - name: Install Rust x86_64-apple-darwin
-        run: rustup target add x86_64-apple-darwin
+      # Dropped x86 macOS support: we no longer build x86_64 mac artifacts
+      # as Apple stopped support for Intel macs. We build arm64-only mac wheels.
 
-      - name: Build on macOS universal2
+      - name: Build on macOS (arm64)
         shell: bash
         env:
           DEVELOPER_DIR: /Applications/Xcode.app/Contents/Developer
           MACOSX_DEPLOYMENT_TARGET: "10.15"
-          ARCHFLAGS: -arch x86_64 -arch arm64
+          # Build for arm64 only for now. Building universal2 in a single
+          # invocation can cause mac x86_64 assembly / linker errors
+          # (fixup/linker errors in huf_decompress) reported during release
+          # builds. We match the regression mac runner (arm64) for now.
+          ARCHFLAGS: -arch arm64
           PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }}
         run: python setup.py bdist_wheel
 

.gitignore

@@ -183,14 +183,14 @@ hits_split/*.parquet
 clickbench/**
 .github/environment/splunk/docker-compose.yml
 testdata/iceberg/**
-third_party/tktech/simdjson/csimdjson.cpp
+third_party/tktech/simdjson/csimdjson.c*
 opteryx/compiled/list_ops/list_ops.pyx
-opteryx/third_party/abseil/containers.cpp
+opteryx/third_party/abseil/containers.c*
 opteryx/third_party/alantsd/base64.c*
 opteryx/third_party/cyan4973/xxhash.c*
-opteryx/third_party/fastfloat/fast_float.cpp
+opteryx/third_party/fastfloat/fast_float.c*
 opteryx/third_party/fuzzy/soundex.c*
-opteryx/third_party/tktech/csimdjson.cpp
+opteryx/third_party/tktech/csimdjson.c*
 opteryx/third_party/ulfjack/ryu.c*
 opteryx/compiled/joins/joins.pyx
 pyiceberg_catalog.db
@@ -199,8 +199,8 @@ testdata/parquet_tests/185d5a679a475304.parquet
 testdata/parquet_tests/42004e494300.parquet
 testdata/parquet_tests/hits_51.parquet
 .gitignore
-third_party/mabel/rugo/parquet/parquet_reader.cpp
-third_party/mabel/rugo/jsonl/jsonl_reader.cpp
+third_party/mabel/rugo/parquet/parquet_reader.c*
+third_party/mabel/rugo/jsonl/jsonl_reader.c*
 generated_*.pyx
 generated_*.c*
 third_party/mabel/draken/interop/arrow.c*
@@ -219,4 +219,5 @@ tests/draken/performance/.perf_baseline.json
 opteryx/draken/vectors/_hash_api.c*
 third_party/mabel/draken/vectors/interval_vector.c*
 third_party/mabel/draken/morsels/align.c*
-third_party/mabel/draken/compiled/maskops.cpp
+third_party/mabel/draken/compiled/maskops.c*
+**.html

DEVELOPER_GUIDE.md

@@ -161,6 +161,16 @@ make build
 make clean
 ```
 
+
+### macOS support
+
+We no longer build or support Intel (x86_64) macOS binaries. Apple stopped adding support for Intel macs and our CI and releases build ARM64 mac wheels only.
+
+To build mac wheel locally for Apple Silicon (arm64):
+
+```bash
+ARCHFLAGS="-arch arm64" python setup.py bdist_wheel
+```
 ## Development Patterns
 
 ### Adding a New Operator

dev/README.md

@@ -16,6 +16,8 @@ This directory contains scripts and tools used for Opteryx development, building
 ./build-wheels.sh
 ```
 
+Note: macOS releases are arm64-only; Intel/x86_64 macOS wheels are not built by CI or released.
+
 ### Updating Build Counter
 ```bash
 python build_counter.py

dev/build-wheels.sh

@@ -2,6 +2,9 @@
 set -ex
 
 # Install OpenSSL development headers inside the container
+# Note: zstd/snappy are vendored into the project; we should not install
+# zstd-devel/snappy-devel via yum inside the manylinux container (they may
+# not be available on the base image and we compile vendor sources directly).
 yum install -y openssl-devel
 
 # Install Rust (required for building some Python packages with Rust extensions)

opteryx/__version__.py

@@ -1,9 +1,9 @@
 # THIS FILE IS AUTOMATICALLY UPDATED DURING THE BUILD PROCESS
 # DO NOT EDIT THIS FILE DIRECTLY
 
-__build__ = 1926
+__build__ = 1954
 __author__ = "@joocer"
-__version__ = "0.26.2-beta.1926"
+__version__ = "0.26.2-beta.1954"
 
 # Store the version here so:
 # 1) we don't load dependencies by storing it in __init__.py