ci.yml

name: CI

on:
  push:
    branches: [ main, master, develop ]
  pull_request:
    branches: [ main, master, develop ]

# Cancel previous runs on the same PR/branch
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  CARGO_INCREMENTAL: 0

jobs:
  docs:
    name: Docs
    runs-on: ubuntu-latest
    env:
      RUSTDOCFLAGS: "--cfg docsrs -D warnings"
    steps:
      - uses: actions/checkout@v4
      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - name: Install protoc
        run: sudo apt-get install -y protobuf-compiler
      - name: Build documentation
        run: cargo doc --no-deps -p skardi

  test:
    name: Test
    runs-on: ubuntu-latest

    services:
      mysql:
        image: mysql:8.0
        env:
          MYSQL_ROOT_PASSWORD: rootpass
          MYSQL_DATABASE: mydb
          MYSQL_USER: skardi_user
          MYSQL_PASSWORD: skardi_pass
        ports:
          - 3306:3306
        options: >-
          --health-cmd="mysqladmin ping -h 127.0.0.1 -u root -prootpass"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=5

      postgres:
        # `pgvector/pgvector:pg16` is the upstream Postgres 16 image with
        # the pgvector extension pre-built in. Used so the `docs` table
        # can declare an `embedding vector(N)` column and the integration
        # tests can round-trip vector cells through the multi-row VALUES
        # renderer without a separate extension build step.
        image: pgvector/pgvector:pg16
        env:
          POSTGRES_DB: mydb
          POSTGRES_USER: skardi_user
          POSTGRES_PASSWORD: skardi_pass
        ports:
          - 5432:5432
        options: >-
          --health-cmd="pg_isready -U skardi_user -d mydb"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=5

      mongo:
        image: mongo:7.0
        env:
          MONGO_INITDB_ROOT_USERNAME: root
          MONGO_INITDB_ROOT_PASSWORD: rootpass
        ports:
          - 27017:27017
        options: >-
          --health-cmd="mongosh --eval 'db.runCommand({ping:1})' --quiet"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=5

      redis:
        image: redis:7.4
        ports:
          - 6379:6379
        options: >-
          --health-cmd="redis-cli ping"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=5

      seekdb:
        image: oceanbase/seekdb:latest
        env:
          OB_TENANT_PASSWORD: skardi_pass
        ports:
          - 2881:2881
          - 2886:2886
        options: >-
          --health-cmd="obclient -h 127.0.0.1 -P 2881 -u root@sys -e 'SELECT 1' || mysql -h 127.0.0.1 -P 2881 -u root@sys -e 'SELECT 1'"
          --health-interval=30s
          --health-timeout=10s
          --health-retries=20

      clickhouse:
        image: clickhouse/clickhouse-server:24.8
        env:
          CLICKHOUSE_DB: mydb
          CLICKHOUSE_USER: skardi_user
          CLICKHOUSE_PASSWORD: skardi_pass
        ports:
          - 8123:8123
        options: >-
          --health-cmd="wget --no-verbose --tries=1 --spider http://127.0.0.1:8123/ping || exit 1"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=5

      # DynamoDB Local has no in-image shell tooling for a healthcheck, so
      # readiness is polled with the AWS CLI in a dedicated step below.
      dynamodb:
        image: amazon/dynamodb-local:2.5.2
        ports:
          - 8000:8000

    env:
      MYSQL_USER: skardi_user
      MYSQL_PASSWORD: skardi_pass
      PG_USER: skardi_user
      PG_PASSWORD: skardi_pass
      MONGO_USER: root
      MONGO_PASS: rootpass
      SEEKDB_USER: "root@sys"
      SEEKDB_PASSWORD: ""
      INFLUXDB_URL: "http://127.0.0.1:8181"
      INFLUXDB_DATABASE: "metrics"
      CLICKHOUSE_URL: "http://127.0.0.1:8123"
      CLICKHOUSE_DATABASE: "mydb"
      CLICKHOUSE_USER: skardi_user
      CLICKHOUSE_PASSWORD: skardi_pass
      # DynamoDB Local ignores credential values but the AWS SDK requires them
      # to be present. The endpoint is supplied via the context/test config.
      AWS_ACCESS_KEY_ID: dummy
      AWS_SECRET_ACCESS_KEY: dummy
      AWS_DEFAULT_REGION: us-east-1

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler
      - name: Install Rust
        run: rustup toolchain install stable --component llvm-tools-preview
      - name: Install cargo-llvm-cov
        uses: taiki-e/install-action@cargo-llvm-cov
      - name: install nextest
        uses: taiki-e/install-action@nextest
      - name: Free up disk space
        # GitHub-hosted runners ship with ~25 GB free on the smaller VM
        # pool — not enough for the coverage-instrumented `--all-features`
        # link step (LLD crashes with SIGBUS when the build write hits a
        # full filesystem). Remove the bulky preinstalled toolchains and
        # caches to give the linker headroom.
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo rm -rf /usr/local/share/powershell
          sudo rm -rf /usr/local/share/chromium
          sudo rm -rf /usr/local/lib/node_modules
          sudo rm -rf /opt/microsoft
          sudo rm -rf /opt/google
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          sudo docker image prune -af || true
          df -h
      - uses: Swatinem/rust-cache@v2
        with:
          # liteparse-pdfium-sys's build.rs downloads a prebuilt PDFium
          # native library to ~/.cache/pdfium-rs (see docs/documents.md) —
          # outside the paths rust-cache tracks by default (~/.cargo/*,
          # target/). On a cache *hit*, cargo's build-script fingerprinting
          # sees the restored target/ output as still valid and skips
          # re-running build.rs, so it never re-downloads PDFium; the
          # runner then has no libpdfium.so and every documents-feature
          # test that touches PDFium panics. Caching this directory too
          # keeps the download in sync with the rest of the restored cache.
          cache-directories: ~/.cache/pdfium-rs
      - name: Check code format
        run: cargo fmt --all -- --check
      - name: Check the package for errors
        run: cargo check --all
      - name: Execute rust tests
        run: cargo llvm-cov --no-report nextest --all-features

      - name: Install integration test dependencies
        run: |
          wget -qO- https://www.mongodb.org/static/pgp/server-7.0.asc | sudo gpg --dearmor -o /usr/share/keyrings/mongodb-server-7.0.gpg
          echo "deb [ signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list
          sudo apt-get update
          sudo apt-get install -y mongodb-mongosh redis-tools

      - name: Seed MySQL data
        run: |
          mysql -h 127.0.0.1 -u skardi_user -pskardi_pass mydb <<'EOF'
          CREATE TABLE users (
              id INT PRIMARY KEY AUTO_INCREMENT,
              name VARCHAR(100) NOT NULL,
              email VARCHAR(100) UNIQUE NOT NULL
          );
          CREATE TABLE orders (
              id INT PRIMARY KEY AUTO_INCREMENT,
              user_id INT NOT NULL,
              product VARCHAR(100) NOT NULL,
              amount DECIMAL(10, 2) NOT NULL
          );
          CREATE TABLE user_order_stats (
              id INT PRIMARY KEY AUTO_INCREMENT,
              user_id INT NOT NULL,
              user_name VARCHAR(100) NOT NULL,
              user_email VARCHAR(100) NOT NULL,
              total_orders INT NOT NULL,
              total_spent DECIMAL(10, 2) NOT NULL,
              last_order_date VARCHAR(50),
              UNIQUE KEY unique_user (user_id)
          );
          INSERT INTO users (name, email) VALUES
              ('Alice Smith', 'alice@example.com'),
              ('Bob Johnson', 'bob@example.com'),
              ('Carol Williams', 'carol@example.com');
          INSERT INTO orders (user_id, product, amount) VALUES
              (1, 'Laptop', 999.99),
              (2, 'Keyboard', 79.99),
              (3, 'Monitor', 299.99);
          EOF

      - name: Seed PostgreSQL data
        run: |
          PGPASSWORD=skardi_pass psql -h 127.0.0.1 -U skardi_user -d mydb <<'EOF'
          CREATE EXTENSION IF NOT EXISTS vector;
          CREATE TABLE users (
              id SERIAL PRIMARY KEY,
              name VARCHAR(100) NOT NULL,
              email VARCHAR(100) UNIQUE NOT NULL
          );
          CREATE TABLE orders (
              id SERIAL PRIMARY KEY,
              user_id INT NOT NULL,
              product VARCHAR(100) NOT NULL,
              amount DECIMAL(10, 2) NOT NULL
          );
          CREATE TABLE user_order_stats (
              user_id INT PRIMARY KEY,
              user_name VARCHAR(100),
              user_email VARCHAR(100),
              total_orders INT,
              total_spent DECIMAL(10, 2),
              last_order_date VARCHAR(50)
          );
          INSERT INTO users (name, email) VALUES
              ('Alice Smith', 'alice@example.com'),
              ('Bob Johnson', 'bob@example.com'),
              ('Carol Williams', 'carol@example.com');
          INSERT INTO orders (user_id, product, amount) VALUES
              (1, 'Laptop', 999.99),
              (2, 'Keyboard', 79.99),
              (3, 'Monitor', 299.99);
          CREATE TABLE articles (
              id SERIAL PRIMARY KEY,
              title VARCHAR(200) NOT NULL,
              body TEXT NOT NULL,
              category VARCHAR(50) NOT NULL
          );
          INSERT INTO articles (title, body, category) VALUES
              ('Intro to Machine Learning', 'machine learning model training deep neural network supervised algorithms', 'ai'),
              ('Natural Language Processing', 'natural language processing text classification sentiment analysis tokenization', 'ai'),
              ('Database Query Optimization', 'database query optimization indexing performance tuning relational algebra', 'database'),
              ('Deep Learning Advances', 'machine learning classification supervised training model convolutional neural network', 'research'),
              ('Neural Network Architectures', 'deep learning neural network convolutional image recognition transformer attention', 'ai');
          -- pgvector-backed `docs` table — exercises the multi-row VALUES
          -- renderer's nested-array cell shape (`{"rows": [["a", "b", [v1,
          -- v2, v3, v4]], ...]}`) end-to-end. Mirrors the SeekDB `docs`
          -- table schema so the same parameter shape works on both.
          CREATE TABLE docs (
              id        TEXT PRIMARY KEY,
              title     TEXT NOT NULL,
              category  TEXT NOT NULL,
              embedding vector(4) NOT NULL
          );
          EOF

      - name: Seed MongoDB data
        run: |
          mongosh "mongodb://root:rootpass@127.0.0.1:27017/?authSource=admin" <<'EOF'
          use mydb

          db.createCollection("products", {
            validator: {
              $jsonSchema: {
                bsonType: "object",
                required: ["product_id", "name", "price"],
                properties: {
                  product_id: { bsonType: "string" },
                  name: { bsonType: "string" },
                  category: { bsonType: "string" },
                  price: { bsonType: "double" },
                  in_stock: { bsonType: "bool" }
                }
              }
            }
          })

          db.products.insertMany([
            { _id: "PROD001", product_id: "PROD001", name: "Laptop", category: "Electronics", price: 999.99, in_stock: true },
            { _id: "PROD002", product_id: "PROD002", name: "Keyboard", category: "Electronics", price: 79.99, in_stock: true },
            { _id: "PROD003", product_id: "PROD003", name: "Monitor", category: "Electronics", price: 299.99, in_stock: false },
            { _id: "PROD004", product_id: "PROD004", name: "Mouse", category: "Electronics", price: 29.99, in_stock: true },
            { _id: "PROD005", product_id: "PROD005", name: "Desk Chair", category: "Furniture", price: 199.99, in_stock: true }
          ])

          db.createCollection("product_stats", {
            validator: {
              $jsonSchema: {
                bsonType: "object",
                required: ["stat_id"],
                properties: {
                  stat_id: { bsonType: "string" },
                  category: { bsonType: "string" },
                  total_products: { bsonType: "long" },
                  total_value: { bsonType: "double" },
                  avg_price: { bsonType: "double" }
                }
              }
            }
          })

          // ─── Full-text search test collection ─────────────────────────
          db.createCollection("dataset_data_texts")
          db.dataset_data_texts.createIndex(
            { fullTextToken: "text" },
            { default_language: "none" }
          )
          db.dataset_data_texts.insertMany([
            { _id: "data001", dataId: "data001", teamId: "team1", datasetId: "ds1", collectionId: "col1", fullTextToken: "machine learning model training deep neural network" },
            { _id: "data002", dataId: "data002", teamId: "team1", datasetId: "ds1", collectionId: "col1", fullTextToken: "natural language processing text classification sentiment" },
            { _id: "data003", dataId: "data003", teamId: "team1", datasetId: "ds1", collectionId: "col2", fullTextToken: "database query optimization indexing performance tuning" },
            { _id: "data004", dataId: "data004", teamId: "team2", datasetId: "ds2", collectionId: "col3", fullTextToken: "machine learning classification supervised training model" },
            { _id: "data005", dataId: "data005", teamId: "team1", datasetId: "ds1", collectionId: "col1", fullTextToken: "deep learning neural network convolutional image recognition" }
          ])
          EOF

      - name: Seed Redis data
        run: |
          redis-cli -h 127.0.0.1 <<'EOF'
          HSET mydb:products:PROD001 name "Laptop" category "Electronics" price "999.99" in_stock "true"
          HSET mydb:products:PROD002 name "Keyboard" category "Electronics" price "79.99" in_stock "true"
          HSET mydb:products:PROD003 name "Monitor" category "Electronics" price "299.99" in_stock "false"
          HSET mydb:products:PROD004 name "Mouse" category "Electronics" price "29.99" in_stock "true"
          HSET mydb:products:PROD005 name "Desk Chair" category "Furniture" price "199.99" in_stock "true"
          EOF

      - name: Wait for SeekDB to accept connections
        run: |
          for i in $(seq 1 60); do
            if mysql -h 127.0.0.1 -P 2881 -u "root@sys" -e "SELECT 1" >/dev/null 2>&1; then
              echo "SeekDB is ready"
              break
            fi
            echo "Waiting for SeekDB... (attempt $i/60)"
            sleep 5
          done

      - name: Seed SeekDB data
        run: |
          mysql -h 127.0.0.1 -P 2881 -u "root@sys" <<'EOF'
          CREATE DATABASE IF NOT EXISTS mydb;
          USE mydb;
          CREATE TABLE IF NOT EXISTS users (
              id INT PRIMARY KEY AUTO_INCREMENT,
              name VARCHAR(100) NOT NULL,
              email VARCHAR(100) UNIQUE NOT NULL
          );
          CREATE TABLE IF NOT EXISTS orders (
              id INT PRIMARY KEY AUTO_INCREMENT,
              user_id INT NOT NULL,
              product VARCHAR(100) NOT NULL,
              amount DECIMAL(10, 2) NOT NULL
          );
          CREATE TABLE IF NOT EXISTS articles (
              id INT PRIMARY KEY AUTO_INCREMENT,
              title VARCHAR(200) NOT NULL,
              body TEXT NOT NULL,
              category VARCHAR(50) NOT NULL,
              FULLTEXT INDEX ft_body (body) WITH PARSER IK
          );
          CREATE TABLE IF NOT EXISTS docs (
              id INT PRIMARY KEY AUTO_INCREMENT,
              title VARCHAR(200) NOT NULL,
              category VARCHAR(50) NOT NULL,
              embedding VECTOR(4),
              VECTOR INDEX idx_embedding (embedding) WITH (TYPE = HNSW, DISTANCE = L2)
          );
          -- Inner-product variant to verify NEGATIVE_INNER_PRODUCT() + HNSW
          -- actually work end-to-end; L2/COSINE are exercised via the docs table.
          CREATE TABLE IF NOT EXISTS docs_ip (
              id INT PRIMARY KEY AUTO_INCREMENT,
              title VARCHAR(200) NOT NULL,
              embedding VECTOR(4),
              VECTOR INDEX idx_embedding_ip (embedding) WITH (TYPE = HNSW, DISTANCE = INNER_PRODUCT)
          );
          INSERT INTO users (name, email) VALUES
              ('Alice Smith', 'alice@example.com'),
              ('Bob Johnson', 'bob@example.com'),
              ('Carol Williams', 'carol@example.com');
          INSERT INTO orders (user_id, product, amount) VALUES
              (1, 'Laptop', 999.99),
              (2, 'Keyboard', 79.99),
              (3, 'Monitor', 299.99);
          INSERT INTO articles (title, body, category) VALUES
              ('Intro to Machine Learning', 'machine learning model training deep neural network supervised algorithms', 'ai'),
              ('Natural Language Processing', 'natural language processing text classification sentiment analysis', 'ai'),
              ('Database Query Optimization', 'database query optimization indexing performance tuning relational', 'database'),
              ('Deep Learning Advances', 'machine learning classification supervised training model convolutional neural', 'research'),
              ('Neural Network Architectures', 'deep learning neural network convolutional image recognition transformer', 'ai');
          INSERT INTO docs (title, category, embedding) VALUES
              ('doc-a', 'electronics', '[1.0, 0.0, 0.0, 0.0]'),
              ('doc-b', 'electronics', '[0.0, 1.0, 0.0, 0.0]'),
              ('doc-c', 'books',       '[0.0, 0.0, 1.0, 0.0]'),
              ('doc-d', 'electronics', '[1.0, 1.0, 0.0, 0.0]'),
              ('doc-e', 'books',       '[0.5, 0.5, 0.5, 0.5]');
          INSERT INTO docs_ip (title, embedding) VALUES
              ('ip-a', '[1.0, 0.0, 0.0, 0.0]'),
              ('ip-b', '[0.0, 1.0, 0.0, 0.0]'),
              ('ip-c', '[1.0, 1.0, 0.0, 0.0]'),
              ('ip-d', '[0.5, 0.5, 0.5, 0.5]');
          EOF

      - name: Start and seed InfluxDB 3
        # InfluxDB 3 Core needs `influxdb3 serve <args>`, which a GitHub
        # Actions service container can't express (no command override), so
        # run it as a plain container. `--object-store memory` keeps it
        # ephemeral and disk-free; `--without-auth` skips token management for
        # the test fixture.
        run: |
          docker run -d --name influxdb3 -p 8181:8181 \
            influxdb:3-core influxdb3 serve \
              --node-id ci-node0 \
              --object-store memory \
              --without-auth
          echo "Waiting for InfluxDB 3 to become healthy..."
          for i in $(seq 1 60); do
            if curl -fsS http://127.0.0.1:8181/health >/dev/null 2>&1; then
              echo "InfluxDB 3 is ready"
              break
            fi
            echo "Waiting for InfluxDB 3... (attempt $i/60)"
            sleep 2
          done
          curl -fsS -XPOST "http://127.0.0.1:8181/api/v3/configure/database" \
            -H "Content-Type: application/json" -d '{"db": "metrics"}'
          curl -fsS "http://127.0.0.1:8181/api/v3/write_lp?db=metrics&precision=second" \
            --data-binary @- <<'EOF'
          cpu,host=host1,region=us-west usage_user=12.5,usage_system=3.2 1700000000
          cpu,host=host1,region=us-west usage_user=64.1,usage_system=9.8 1700000060
          cpu,host=host2,region=us-west usage_user=41.0,usage_system=6.0 1700000000
          cpu,host=host2,region=us-west usage_user=88.7,usage_system=12.3 1700000060
          cpu,host=host3,region=us-east usage_user=22.4,usage_system=4.1 1700000000
          mem,host=host1,region=us-west used_percent=48.2 1700000000
          mem,host=host2,region=us-west used_percent=73.9 1700000000
          mem,host=host3,region=us-east used_percent=31.5 1700000000
          EOF

      - name: Seed ClickHouse data
        # The HTTP interface accepts one statement per request, so the seed
        # script is split on blank lines and posted statement by statement.
        run: |
          seed() {
            curl -fsS "http://127.0.0.1:8123/?user=skardi_user&password=skardi_pass" \
              --data-binary "$1" > /dev/null
          }
          seed "CREATE TABLE mydb.users (
              id UInt32,
              name String,
              email String
          ) ENGINE = MergeTree ORDER BY id"
          seed "INSERT INTO mydb.users VALUES
              (1, 'Alice Smith', 'alice@example.com'),
              (2, 'Bob Johnson', 'bob@example.com'),
              (3, 'Carol Williams', 'carol@example.com')"
          seed "CREATE TABLE mydb.orders (
              id UInt32,
              user_id UInt32,
              product String,
              amount Float64
          ) ENGINE = MergeTree ORDER BY id"
          seed "INSERT INTO mydb.orders VALUES
              (1, 1, 'Laptop', 999.99),
              (2, 2, 'Keyboard', 79.99),
              (3, 3, 'Monitor', 299.99)"
          seed "CREATE TABLE mydb.products (
              product_id String,
              name String,
              category Nullable(String),
              price Float64,
              in_stock Bool
          ) ENGINE = MergeTree ORDER BY product_id"
          # PROD005 has a NULL category on purpose — exercises NULL handling.
          seed "INSERT INTO mydb.products VALUES
              ('PROD001', 'Laptop', 'Electronics', 999.99, true),
              ('PROD002', 'Keyboard', 'Electronics', 79.99, true),
              ('PROD003', 'Monitor', 'Electronics', 299.99, false),
              ('PROD004', 'Mouse', 'Electronics', 29.99, true),
              ('PROD005', 'Desk Chair', NULL, 199.99, true)"
          # Deliberately left empty — exercises schema inference on empty tables.
          seed "CREATE TABLE mydb.empty_metrics (
              ts DateTime,
              value Float64
          ) ENGINE = MergeTree ORDER BY ts"

      - name: Seed DynamoDB data
        env:
          EP: http://127.0.0.1:8000
        run: |
          # Wait for DynamoDB Local to accept connections.
          for i in $(seq 1 30); do
            if aws dynamodb list-tables --endpoint-url "$EP" >/dev/null 2>&1; then
              echo "DynamoDB Local is ready"
              break
            fi
            echo "Waiting for DynamoDB Local... (attempt $i/30)"
            sleep 2
          done

          aws dynamodb create-table --endpoint-url "$EP" \
            --table-name products \
            --attribute-definitions AttributeName=product_id,AttributeType=S \
            --key-schema AttributeName=product_id,KeyType=HASH \
            --billing-mode PAY_PER_REQUEST
          aws dynamodb wait table-exists --endpoint-url "$EP" --table-name products

          put() { aws dynamodb put-item --endpoint-url "$EP" --table-name products --item "$1"; }
          put '{"product_id":{"S":"PROD001"},"name":{"S":"Laptop"},"category":{"S":"Electronics"},"price":{"N":"999.99"},"in_stock":{"BOOL":true}}'
          put '{"product_id":{"S":"PROD002"},"name":{"S":"Keyboard"},"category":{"S":"Electronics"},"price":{"N":"79.99"},"in_stock":{"BOOL":true}}'
          put '{"product_id":{"S":"PROD003"},"name":{"S":"Monitor"},"category":{"S":"Electronics"},"price":{"N":"299.99"},"in_stock":{"BOOL":false}}'
          put '{"product_id":{"S":"PROD004"},"name":{"S":"Mouse"},"category":{"S":"Electronics"},"price":{"N":"29.99"},"in_stock":{"BOOL":true}}'
          # NULL-bearing row: no `category` attribute, exercising NULL handling.
          put '{"product_id":{"S":"PROD005"},"name":{"S":"Desk Chair"},"price":{"N":"199.99"},"in_stock":{"BOOL":true}}'

      - name: Start skardi-server for CLI e2e smoke tests
        run: |
          # The CLI's #[ignore] e2e_smoke tests are integration tests whose
          # backing service is skardi-server itself; they hit the default
          # http://127.0.0.1:8080 (no ctx/pipelines needed). The nextest step
          # above already compiled the server binary into the llvm-cov
          # target dir, so this starts without rebuilding.
          ./target/llvm-cov-target/debug/skardi-server --port 8080 &
          for i in $(seq 1 30); do
            curl -sf http://127.0.0.1:8080/health >/dev/null && exit 0
            sleep 1
          done
          echo "skardi-server failed to become healthy" >&2
          exit 1

      - name: Execute Integration tests
        run: cargo llvm-cov --no-report nextest --all-features -- --ignored

      # MinIO gives the `documents` connector's object-store path real S3
      # coverage without needing an AWS account or credentials in CI. Started
      # with `docker run` rather than a `services:` entry because service
      # containers cannot override the image command, and the MinIO image needs
      # `server /data`.
      - name: Start MinIO for documents S3 tests
        run: |
          docker run -d --name skardi-minio \
            -p 127.0.0.1:9000:9000 \
            -e MINIO_ROOT_USER=skardiminio \
            -e MINIO_ROOT_PASSWORD=skardiminio123 \
            quay.io/minio/minio server /data
          # The image has no shell tooling for a container healthcheck, so poll
          # readiness from the runner (same approach as DynamoDB Local above).
          for i in $(seq 1 30); do
            if curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null 2>&1; then
              echo "minio ready after ${i}s"
              break
            fi
            if [ "$i" = "30" ]; then
              echo "minio failed to become ready"
              docker logs skardi-minio
              exit 1
            fi
            sleep 1
          done
          AWS_ACCESS_KEY_ID=skardiminio AWS_SECRET_ACCESS_KEY=skardiminio123 \
            aws --endpoint-url http://127.0.0.1:9000 s3 mb s3://skardi-ci-documents

      # These are `#[ignore]`d and skip themselves unless DOCUMENTS_S3_LIVE=1, so
      # the step above's credentials are what actually arms them. Set at step
      # scope, not job scope: the job-level AWS_* values are the `dummy`
      # placeholders DynamoDB Local expects, and MinIO rejects a root password
      # shorter than 8 characters.
      - name: Execute documents S3 live tests against MinIO
        env:
          DOCUMENTS_S3_LIVE: "1"
          DOCUMENTS_S3_BUCKET: skardi-ci-documents
          # `AmazonS3Builder::from_env()` honours both of these, so the real S3
          # HTTP client path is exercised, not a stub.
          AWS_ENDPOINT: http://127.0.0.1:9000
          AWS_ALLOW_HTTP: "true"
          AWS_REGION: us-east-1
          AWS_ACCESS_KEY_ID: skardiminio
          AWS_SECRET_ACCESS_KEY: skardiminio123
        # `--no-tests=fail` is explicit rather than relying on the current
        # default: if these tests are ever renamed out of the filter's reach, the
        # step must fail loudly instead of reporting success having run nothing.
        run: |
          cargo llvm-cov --no-report nextest --all-features \
            -E 'test(/live_s3_/)' --no-tests=fail -- --ignored

      - name: Stop MinIO
        if: always()
        run: docker rm -f skardi-minio || true

      - name: Generate coverage report (lcov)
        run: cargo llvm-cov report --lcov --output-path lcov.info

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v5
        with:
          files: lcov.info
          fail_ci_if_error: false
          token: ${{ secrets.CODECOV_TOKEN }}