Multi-shard search: Rust merges #6613

Workflow file for this run

	name: CI

	on:
	pull_request:
	branches:
	- main
	- dev
	push:
	branches:
	- main
	- dev

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	# Pre-checks
	nidx-format-rust:
	name: "nidx: Rust code Format"
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	- run: rustup component add rustfmt
	- name: rustfmt
	run: cd nidx && cargo fmt --check --all

	nidx-clippy-rust:
	name: "nidx: Clippy lint"
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	- run: rustup component add clippy
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	workspaces: nidx
	- name: Install Protoc
	uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- name: clippy
	run: cd nidx && cargo clippy --workspace --all-targets --all-features
	env:
	RUSTFLAGS: "-Dwarnings"

	pre-checks-python:
	name: "Pre-checks: Python lint"
	runs-on: ubuntu-24.04
	needs:
	- build-virtual-env
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.14.2"
	- name: Restore venv
	uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Lint source code
	run: \|
	source .venv/bin/activate
	for package in nucliadb nucliadb_utils nucliadb_telemetry nucliadb_sdk nucliadb_dataset nucliadb_models
	do
	make -C $package lint \|\| ERROR=1
	done
	exit $ERROR

	pre-checks-python-old:
	name: "Pre-checks: Python lint 3.10"
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	workspaces: nidx
	- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.10.19"
	- name: Install uv
	uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
	# Needed for nidx_protos (rust)
	- name: Install Protoc
	uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}

	- name: Install venv
	uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0
	with:
	max_attempts: 2
	retry_on: error
	timeout_minutes: 20
	command: uv sync --no-editable --no-group nidx
	- name: Clean build
	run: find . -name build \| xargs rm -r \|\| true
	- name: Lint source code
	run: \|
	source .venv/bin/activate
	for package in nucliadb nucliadb_utils nucliadb_telemetry nucliadb_sdk nucliadb_dataset nucliadb_models
	do
	make -C $package lint \|\| ERROR=1
	done
	exit $ERROR
	- name: Validate Python
	run: \|
	source .venv/bin/activate
	for package in nucliadb nucliadb_utils nucliadb_telemetry nucliadb_sdk nucliadb_dataset nucliadb_models
	do
	make -C $package tests pytest_extra_flags="--collect-only"
	done
	exit $ERROR

	lint-helm:
	name: Lint all helm charts
	runs-on: ubuntu-24.04

	steps:
	- name: Checkout repository
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

	- name: Install Helm
	uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0
	with:
	version: v3.15.3

	- name: Lint helm packages
	run: \|-
	helm lint charts/*
	helm package charts/*

	check-licenses:
	name: Check Licenses
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	- name: Check License Header
	uses: apache/skywalking-eyes/header@61275cc80d0798a405cb070f7d3a8aaf7cf2c2c1 # v0.8.0
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20
	name: cargo deny
	with:
	log-level: warn
	command: check licenses
	manifest-path: nidx/Cargo.toml

	check-migrations:
	runs-on: ubuntu-24.04
	needs:
	- build-virtual-env
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	- name: Setup Python
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.14.2"
	- name: Restore venv
	uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Check migrations
	run: .venv/bin/nucliadb-validate-migrations

	nidx-tests:
	name: nidx tests
	runs-on: ubuntu-24.04
	services:
	postgres:
	image: postgres:15
	env:
	POSTGRES_PASSWORD: postgres
	ports:
	- 5432:5432
	options: >-
	--health-cmd "pg_isready -U postgres"
	--health-interval 1s
	--health-timeout 5s
	--health-retries 10
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	workspaces: nidx
	cache-all-crates: true # To cache cargo-llvm-cov
	- name: Install Protoc
	uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- name: Install cargo-llvm-cov and nextest
	run: cargo install cargo-llvm-cov@0.6.17 cargo-nextest@0.9.100
	- name: test
	env:
	DATABASE_URL: postgresql://postgres:postgres@localhost/postgres
	SQLX_OFFLINE: true
	run: cd nidx && cargo llvm-cov --codecov --output-path codecov.json nextest --workspace --all-features
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
	with:
	file: ./nidx/codecov.json
	flags: nidx
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	# Builds
	build-nidx-binary:
	name: Build nidx binary
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	workspaces: nidx
	- name: Install Protoc
	uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- name: Compile
	run: cd nidx && cargo build --release
	- name: Upload binaries to cache
	uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	path: "nidx/target/release/nidx"
	key: nidx-bin-${{ github.sha }}

	build-virtual-env:
	name: Build virtual environment
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	with:
	workspaces: nidx
	- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.14.2"
	- name: Install uv
	uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0

	- name: Check uv.lock
	run: uv lock --check

	# Needed for nidx_protos (rust)
	- name: Install Protoc
	uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}

	- name: Install venv
	uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0
	with:
	max_attempts: 2
	retry_on: error
	timeout_minutes: 20
	command: uv sync --no-editable
	- name: Upload virtualenv to cache
	uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	path: .venv
	key: venv-${{ github.sha }}

	# Tests
	nucliadb-search-tests:
	name: NucliaDB search tests
	runs-on: ubuntu-24.04
	needs:
	- build-nidx-binary
	- build-virtual-env
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	- name: Setup Python
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.14.2"
	- name: Restore venv
	uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Restore nidx binary
	uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	path: "nidx/target/release/nidx"
	key: nidx-bin-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Build docker images for nidx
	run: mkdir builds && mv nidx/target/release/nidx builds && docker build -t nidx -f Dockerfile.nidx_prebuilt .

	- name: Run tests
	run: source .venv/bin/activate && make -C nucliadb test-cov-search

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
	with:
	file: ./nucliadb/coverage.xml
	flags: nucliadb-search
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	nucliadb-tests:
	name: NucliaDB tests
	runs-on: ubuntu-24.04
	needs:
	- build-virtual-env
	strategy:
	fail-fast: false
	matrix:
	shard: [0, 1, 2]

	services:
	postgres:
	image: postgres:13
	env:
	POSTGRES_PASSWORD: postgres
	options: >-
	--health-cmd pg_isready
	--health-interval 5s
	--health-timeout 1s
	--health-retries 5
	ports:
	- 5432:5432

	steps:
	- name: Checkout the repository
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

	- name: Setup Python
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.14.2"

	- name: Restore venv
	uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Run NucliaDB tests
	env:
	TESTING_PG_URL: "postgresql://postgres:postgres@localhost/postgres"
	run: \|
	source .venv/bin/activate &&
	make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 -n 4" \|\|
	make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 --lf"

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
	with:
	file: ./nucliadb/coverage.xml
	flags: nucliadb
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	nucliadb-components-tests:
	name: NucliaDB components tests
	runs-on: ubuntu-24.04
	needs:
	- build-virtual-env
	strategy:
	fail-fast: false
	matrix:
	component:
	- ingest
	- reader
	- standalone
	- train
	- writer

	steps:
	- name: Checkout the repository
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

	- name: Setup Python
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.14.2"

	- name: Restore venv
	uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Run package tests
	run: source .venv/bin/activate && make -C nucliadb test-cov-${{ matrix.component }}

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
	with:
	file: ./nucliadb/coverage.xml
	flags: nucliadb-${{ matrix.component }}
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	python-packages-tests:
	name: Python packages tests
	runs-on: ubuntu-24.04
	needs:
	- build-virtual-env
	strategy:
	fail-fast: false
	matrix:
	package:
	- nucliadb_dataset
	- nucliadb_models
	- nucliadb_sdk
	- nucliadb_telemetry
	- nucliadb_utils

	steps:
	- name: Checkout the repository
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

	- name: Setup Python
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.14.2"

	- name: Restore venv
	uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Run package tests
	run: source .venv/bin/activate && make -C ${{ matrix.package }} test-cov

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
	with:
	file: ./${{ matrix.package }}/coverage.xml
	flags: ${{ matrix.package }}
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	# Depends on all tests to pass
	ci-passed:
	name: CI passed
	runs-on: ubuntu-24.04
	needs:
	- nidx-format-rust
	- nidx-clippy-rust
	- pre-checks-python
	- check-licenses
	- check-migrations
	- lint-helm
	- nucliadb-search-tests
	- nucliadb-tests
	- nucliadb-components-tests
	- python-packages-tests
	- nidx-tests
	steps:
	- name: CI passed
	run: echo "CI passed"

	deploy:
	name: Trigger deploy
	if: github.event_name == 'push'
	needs:
	- ci-passed
	uses: ./.github/workflows/deploy.yml
	secrets: inherit
	with:
	environment: ${{ github.ref == 'refs/heads/main' && 'stage' \|\| 'dev' }}

	release:
	name: Trigger release
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	needs:
	- ci-passed
	uses: ./.github/workflows/release.yml
	secrets: inherit

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Multi-shard search: Rust merges #6613

Workflow file

Multi-shard search: Rust merges #6613

Uh oh!

Workflow file for this run