Run the integration test suite with various VectorIO providers #14559
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Vector IO Integration Tests | |
| run-name: Run the integration test suite with various VectorIO providers | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| pull_request: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| paths: | |
| - 'src/ogx/**' | |
| - '!src/ogx_ui/**' | |
| - 'tests/integration/vector_io/**' | |
| - 'tests/integration/conftest.py' # Test configuration that affects vector IO tests | |
| - 'uv.lock' | |
| - 'pyproject.toml' | |
| - 'requirements.txt' | |
| - '.github/workflows/integration-vector-io-tests.yml' # This workflow | |
| merge_group: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| schedule: | |
| - cron: '0 0 * * 1' # (test on python 3.13) Weekly on Monday at 12 AM UTC | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test-matrix: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| # Use a smaller set of providers on PRs to reduce runner contention | |
| vector-io-provider: ${{ github.event_name == 'pull_request' && fromJSON('["inline::faiss", "inline::sqlite-vec", "remote::pgvector", "remote::neo4j"]') || fromJSON('["inline::faiss", "inline::sqlite-vec", "inline::milvus", "remote::chromadb", "remote::pgvector", "remote::weaviate", "remote::qdrant", "remote::elasticsearch", "remote::infinispan", "remote::neo4j"]') }} | |
| python-version: ${{ github.event.schedule == '0 0 * * 1' && fromJSON('["3.12", "3.13"]') || fromJSON('["3.12"]') }} | |
| fail-fast: false # we want to run all tests regardless of failure | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Install dependencies | |
| uses: ./.github/actions/setup-runner | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup Chroma | |
| if: matrix.vector-io-provider == 'remote::chromadb' | |
| run: | | |
| docker run --rm -d --pull always \ | |
| --name chromadb \ | |
| -p 8000:8000 \ | |
| -v ~/chroma:/chroma/chroma \ | |
| -e IS_PERSISTENT=TRUE \ | |
| -e ANONYMIZED_TELEMETRY=FALSE \ | |
| chromadb/chroma:latest | |
| - name: Setup Weaviate | |
| if: matrix.vector-io-provider == 'remote::weaviate' | |
| run: | | |
| docker run --rm -d --pull always \ | |
| --name weaviate \ | |
| -p 8080:8080 -p 50051:50051 \ | |
| cr.weaviate.io/semitechnologies/weaviate:1.32.0 | |
| - name: Start PGVector DB | |
| if: matrix.vector-io-provider == 'remote::pgvector' | |
| run: | | |
| docker run -d \ | |
| --name pgvector \ | |
| -e POSTGRES_USER=ogx \ | |
| -e POSTGRES_PASSWORD=ogx \ | |
| -e POSTGRES_DB=ogx \ | |
| -p 5432:5432 \ | |
| pgvector/pgvector:pg17 | |
| - name: Start Neo4j | |
| if: matrix.vector-io-provider == 'remote::neo4j' | |
| run: | | |
| docker run --rm -d \ | |
| --name neo4j \ | |
| -p 7474:7474 -p 7687:7687 \ | |
| -e NEO4J_AUTH=neo4j/ogxpassword \ | |
| neo4j:2025.10 | |
| - name: Wait for Neo4j to be ready | |
| if: matrix.vector-io-provider == 'remote::neo4j' | |
| run: | | |
| echo "Waiting for Neo4j to be ready..." | |
| for _ in {1..60}; do | |
| if docker exec neo4j cypher-shell -u neo4j -p ogxpassword 'RETURN 1 AS ok' >/dev/null 2>&1; then | |
| echo "Neo4j is ready!" | |
| exit 0 | |
| fi | |
| sleep 2 | |
| done | |
| echo "Neo4j failed to start" | |
| docker logs neo4j | |
| exit 1 | |
| - name: Wait for PGVector to be ready | |
| if: matrix.vector-io-provider == 'remote::pgvector' | |
| run: | | |
| echo "Waiting for Postgres to be ready..." | |
| for i in {1..30}; do | |
| if docker exec pgvector pg_isready -U ogx -d ogx > /dev/null 2>&1; then | |
| echo "Postgres is ready after ${i}s!" | |
| break | |
| fi | |
| echo "Not ready yet... (${i})" | |
| sleep 1 | |
| done | |
| - name: Enable pgvector extension | |
| if: matrix.vector-io-provider == 'remote::pgvector' | |
| run: | | |
| for i in {1..10}; do | |
| if PGPASSWORD=ogx psql -h localhost -U ogx -d ogx \ | |
| -c "CREATE EXTENSION IF NOT EXISTS vector;" 2>/dev/null; then | |
| echo "pgvector extension enabled" | |
| exit 0 | |
| fi | |
| echo "psql not ready yet, retrying... (${i})" | |
| sleep 2 | |
| done | |
| echo "Failed to enable pgvector extension after retries" | |
| exit 1 | |
| - name: Setup Qdrant | |
| if: matrix.vector-io-provider == 'remote::qdrant' | |
| run: | | |
| docker run --rm -d --pull always \ | |
| --name qdrant \ | |
| -p 6333:6333 \ | |
| qdrant/qdrant | |
| - name: Wait for Qdrant to be ready | |
| if: matrix.vector-io-provider == 'remote::qdrant' | |
| run: | | |
| echo "Waiting for Qdrant to be ready..." | |
| for _ in {1..30}; do | |
| if curl -s http://localhost:6333/collections | grep -q '"status":"ok"'; then | |
| echo "Qdrant is ready!" | |
| exit 0 | |
| fi | |
| sleep 2 | |
| done | |
| echo "Qdrant failed to start" | |
| docker logs qdrant | |
| exit 1 | |
| - name: Wait for ChromaDB to be ready | |
| if: matrix.vector-io-provider == 'remote::chromadb' | |
| run: | | |
| echo "Waiting for ChromaDB to be ready..." | |
| for _ in {1..30}; do | |
| if curl -s http://localhost:8000/api/v2/heartbeat | grep -q "nanosecond heartbeat"; then | |
| echo "ChromaDB is ready!" | |
| exit 0 | |
| fi | |
| sleep 2 | |
| done | |
| echo "ChromaDB failed to start" | |
| docker logs chromadb | |
| exit 1 | |
| - name: Wait for Weaviate to be ready | |
| if: matrix.vector-io-provider == 'remote::weaviate' | |
| run: | | |
| echo "Waiting for Weaviate to be ready..." | |
| for _ in {1..30}; do | |
| if curl -s http://localhost:8080 | grep -q "https://weaviate.io/developers/weaviate/current/"; then | |
| echo "Weaviate is ready!" | |
| exit 0 | |
| fi | |
| sleep 2 | |
| done | |
| echo "Weaviate failed to start" | |
| docker logs weaviate | |
| exit 1 | |
| - name: Setup Elasticsearch | |
| if: matrix.vector-io-provider == 'remote::elasticsearch' | |
| id: setup-elasticsearch | |
| run: | | |
| curl -fsSL https://elastic.co/start-local | sh -s -- -v 9.2.0 --esonly | |
| source elastic-start-local/.env | |
| echo "ELASTICSEARCH_API_KEY=$ES_LOCAL_API_KEY" >> "$GITHUB_ENV" | |
| - name: Setup Infinispan | |
| if: matrix.vector-io-provider == 'remote::infinispan' | |
| run: | | |
| docker run --rm -d --pull always \ | |
| --name infinispan \ | |
| -p 11222:11222 \ | |
| -e USER=admin \ | |
| -e PASS=password \ | |
| infinispan/server:16.1.0 | |
| - name: Wait for Infinispan to be ready | |
| if: matrix.vector-io-provider == 'remote::infinispan' | |
| run: | | |
| echo "Waiting for Infinispan to be ready..." | |
| for _ in {1..30}; do | |
| if curl -s -u admin:password http://localhost:11222/rest/v2/server | grep -q "version"; then | |
| echo "Infinispan is ready!" | |
| exit 0 | |
| fi | |
| sleep 2 | |
| done | |
| echo "Infinispan failed to start" | |
| docker logs infinispan | |
| exit 1 | |
| - name: Cache Hugging Face models | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: ~/.cache/huggingface | |
| key: hf-${{ runner.os }}-${{ matrix.python-version }}-nomic-embed-text-v1.5 | |
| - name: Build OGX | |
| run: | | |
| uv run --no-sync ogx stack list-deps ci-tests | xargs -L1 uv pip install | |
| - name: Pre-download embedding model | |
| run: | | |
| uv run --no-sync python - <<'PY' | |
| from sentence_transformers import SentenceTransformer | |
| SentenceTransformer("nomic-ai/nomic-embed-text-v1.5", trust_remote_code=True) | |
| PY | |
| - name: Check Storage and Memory Available Before Tests | |
| if: ${{ always() }} | |
| run: | | |
| free -h | |
| df -h | |
| - name: Run Vector IO Integration Tests | |
| env: | |
| HF_HUB_OFFLINE: "1" | |
| TRANSFORMERS_OFFLINE: "1" | |
| ENABLE_CHROMADB: ${{ matrix.vector-io-provider == 'remote::chromadb' && 'true' || '' }} | |
| CHROMADB_URL: ${{ matrix.vector-io-provider == 'remote::chromadb' && 'http://localhost:8000' || '' }} | |
| ENABLE_PGVECTOR: ${{ matrix.vector-io-provider == 'remote::pgvector' && 'true' || '' }} | |
| PGVECTOR_HOST: ${{ matrix.vector-io-provider == 'remote::pgvector' && 'localhost' || '' }} | |
| PGVECTOR_PORT: ${{ matrix.vector-io-provider == 'remote::pgvector' && '5432' || '' }} | |
| PGVECTOR_DB: ${{ matrix.vector-io-provider == 'remote::pgvector' && 'ogx' || '' }} | |
| PGVECTOR_USER: ${{ matrix.vector-io-provider == 'remote::pgvector' && 'ogx' || '' }} | |
| PGVECTOR_PASSWORD: ${{ matrix.vector-io-provider == 'remote::pgvector' && 'ogx' || '' }} | |
| ENABLE_QDRANT: ${{ matrix.vector-io-provider == 'remote::qdrant' && 'true' || '' }} | |
| QDRANT_URL: ${{ matrix.vector-io-provider == 'remote::qdrant' && 'http://localhost:6333' || '' }} | |
| ENABLE_WEAVIATE: ${{ matrix.vector-io-provider == 'remote::weaviate' && 'true' || '' }} | |
| WEAVIATE_CLUSTER_URL: ${{ matrix.vector-io-provider == 'remote::weaviate' && 'localhost:8080' || '' }} | |
| ELASTICSEARCH_URL: ${{ matrix.vector-io-provider == 'remote::elasticsearch' && 'http://localhost:9200' || '' }} | |
| ENABLE_INFINISPAN: ${{ matrix.vector-io-provider == 'remote::infinispan' && 'true' || '' }} | |
| INFINISPAN_URL: ${{ matrix.vector-io-provider == 'remote::infinispan' && 'http://localhost:11222' || '' }} | |
| INFINISPAN_USERNAME: ${{ matrix.vector-io-provider == 'remote::infinispan' && 'admin' || '' }} | |
| INFINISPAN_PASSWORD: ${{ matrix.vector-io-provider == 'remote::infinispan' && 'password' || '' }} | |
| NEO4J_URI: ${{ matrix.vector-io-provider == 'remote::neo4j' && 'bolt://localhost:7687' || '' }} | |
| NEO4J_USER: ${{ matrix.vector-io-provider == 'remote::neo4j' && 'neo4j' || '' }} | |
| NEO4J_PASSWORD: ${{ matrix.vector-io-provider == 'remote::neo4j' && 'ogxpassword' || '' }} | |
| NEO4J_DATABASE: ${{ matrix.vector-io-provider == 'remote::neo4j' && 'neo4j' || '' }} | |
| run: | | |
| uv run --no-sync \ | |
| pytest -sv --stack-config="files=inline::localfs,inference=inline::sentence-transformers?trust_remote_code=true,vector_io=${{ matrix.vector-io-provider }},file_processors=inline::pypdf" \ | |
| tests/integration/vector_io | |
| - name: Check Storage and Memory Available After Tests | |
| if: ${{ always() }} | |
| run: | | |
| free -h | |
| df -h | |
| - name: Create sanitized provider name | |
| if: ${{ always() }} | |
| run: | | |
| echo "SANITIZED_PROVIDER=$(echo "${{ matrix.vector-io-provider }}" | tr ':' '_')" >> "$GITHUB_ENV" | |
| - name: Write ChromaDB logs to file | |
| if: ${{ always() && matrix.vector-io-provider == 'remote::chromadb' }} | |
| run: | | |
| docker logs chromadb > chromadb.log | |
| - name: Write Qdrant logs to file | |
| if: ${{ always() && matrix.vector-io-provider == 'remote::qdrant' }} | |
| run: | | |
| docker logs qdrant > qdrant.log | |
| - name: Write Elasticsearch logs to file | |
| if: ${{ always() && matrix.vector-io-provider == 'remote::elasticsearch' }} | |
| run: | | |
| docker logs es-local-dev > elasticsearch.log | |
| - name: Write Infinispan logs to file | |
| if: ${{ always() && matrix.vector-io-provider == 'remote::infinispan' }} | |
| run: | | |
| docker logs infinispan > infinispan.log | |
| - name: Write Neo4j logs to file | |
| if: ${{ always() && matrix.vector-io-provider == 'remote::neo4j' }} | |
| run: | | |
| docker logs neo4j > neo4j.log | |
| - name: Upload all logs to artifacts | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: vector-io-logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ env.SANITIZED_PROVIDER }}-${{ matrix.python-version }} | |
| path: | | |
| *.log | |
| retention-days: 1 |