feat(sources): Gmail source pack (milestone 5.4) #758

Workflow file for this run

	name: CI

	on:
	push:
	branches: [ main, master, develop ]
	pull_request:
	branches: [ main, master, develop ]

	# Cancel previous runs on the same PR/branch
	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	env:
	CARGO_TERM_COLOR: always
	CARGO_INCREMENTAL: 0

	jobs:
	docs:
	name: Docs
	runs-on: ubuntu-latest
	env:
	RUSTDOCFLAGS: "--cfg docsrs -D warnings"
	steps:
	- uses: actions/checkout@v4
	- name: Install Rust toolchain
	# No toolchain argument: rustup installs the version pinned in
	# rust-toolchain.toml, keeping CI in lockstep with the Docker build.
	run: rustup toolchain install
	- uses: Swatinem/rust-cache@v2
	- name: Install protoc
	run: sudo apt-get install -y protobuf-compiler
	- name: Build documentation
	run: cargo doc --no-deps -p skardi

	test:
	name: Test
	runs-on: ubuntu-latest

	services:
	mysql:
	image: mysql:8.0
	env:
	MYSQL_ROOT_PASSWORD: rootpass
	MYSQL_DATABASE: mydb
	MYSQL_USER: skardi_user
	MYSQL_PASSWORD: skardi_pass
	ports:
	- 3306:3306
	options: >-
	--health-cmd="mysqladmin ping -h 127.0.0.1 -u root -prootpass"
	--health-interval=10s
	--health-timeout=5s
	--health-retries=5

	postgres:
	# `pgvector/pgvector:pg16` is the upstream Postgres 16 image with
	# the pgvector extension pre-built in. Used so the `docs` table
	# can declare an `embedding vector(N)` column and the integration
	# tests can round-trip vector cells through the multi-row VALUES
	# renderer without a separate extension build step.
	image: pgvector/pgvector:pg16
	env:
	POSTGRES_DB: mydb
	POSTGRES_USER: skardi_user
	POSTGRES_PASSWORD: skardi_pass
	ports:
	- 5432:5432
	options: >-
	--health-cmd="pg_isready -U skardi_user -d mydb"
	--health-interval=10s
	--health-timeout=5s
	--health-retries=5

	mongo:
	image: mongo:7.0
	env:
	MONGO_INITDB_ROOT_USERNAME: root
	MONGO_INITDB_ROOT_PASSWORD: rootpass
	ports:
	- 27017:27017
	options: >-
	--health-cmd="mongosh --eval 'db.runCommand({ping:1})' --quiet"
	--health-interval=10s
	--health-timeout=5s
	--health-retries=5

	redis:
	image: redis:7.4
	ports:
	- 6379:6379
	options: >-
	--health-cmd="redis-cli ping"
	--health-interval=10s
	--health-timeout=5s
	--health-retries=5

	seekdb:
	image: oceanbase/seekdb:latest
	env:
	OB_TENANT_PASSWORD: skardi_pass
	ports:
	- 2881:2881
	- 2886:2886
	options: >-
	--health-cmd="obclient -h 127.0.0.1 -P 2881 -u root@sys -e 'SELECT 1' \|\| mysql -h 127.0.0.1 -P 2881 -u root@sys -e 'SELECT 1'"
	--health-interval=30s
	--health-timeout=10s
	--health-retries=20

	clickhouse:
	image: clickhouse/clickhouse-server:24.8
	env:
	CLICKHOUSE_DB: mydb
	CLICKHOUSE_USER: skardi_user
	CLICKHOUSE_PASSWORD: skardi_pass
	ports:
	- 8123:8123
	options: >-
	--health-cmd="wget --no-verbose --tries=1 --spider http://127.0.0.1:8123/ping \|\| exit 1"
	--health-interval=10s
	--health-timeout=5s
	--health-retries=5

	# DynamoDB Local has no in-image shell tooling for a healthcheck, so
	# readiness is polled with the AWS CLI in a dedicated step below.
	dynamodb:
	image: amazon/dynamodb-local:2.5.2
	ports:
	- 8000:8000

	env:
	MYSQL_USER: skardi_user
	MYSQL_PASSWORD: skardi_pass
	PG_USER: skardi_user
	PG_PASSWORD: skardi_pass
	MONGO_USER: root
	MONGO_PASS: rootpass
	SEEKDB_USER: "root@sys"
	SEEKDB_PASSWORD: ""
	INFLUXDB_URL: "http://127.0.0.1:8181"
	INFLUXDB_DATABASE: "metrics"
	CLICKHOUSE_URL: "http://127.0.0.1:8123"
	CLICKHOUSE_DATABASE: "mydb"
	CLICKHOUSE_USER: skardi_user
	CLICKHOUSE_PASSWORD: skardi_pass
	# DynamoDB Local ignores credential values but the AWS SDK requires them
	# to be present. The endpoint is supplied via the context/test config.
	AWS_ACCESS_KEY_ID: dummy
	AWS_SECRET_ACCESS_KEY: dummy
	AWS_DEFAULT_REGION: us-east-1

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Install system dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y protobuf-compiler
	- name: Install Rust
	# Version comes from rust-toolchain.toml. Runners install the pinned
	# toolchain with the minimal profile, and a bare `rustup toolchain
	# install` ignores CLI --component flags in favor of the toolchain
	# file's spec — so components must be added in a second command.
	# Kept out of the toolchain file so the Docker build doesn't pull
	# them: rustfmt is for `cargo fmt`, llvm-tools-preview for coverage.
	run: \|
	rustup toolchain install
	rustup component add rustfmt llvm-tools-preview
	- name: Install cargo-llvm-cov
	uses: taiki-e/install-action@cargo-llvm-cov
	- name: install nextest
	uses: taiki-e/install-action@nextest
	- name: Free up disk space
	# GitHub-hosted runners ship with ~25 GB free on the smaller VM
	# pool — not enough for the coverage-instrumented `--all-features`
	# link step (LLD crashes with SIGBUS when the build write hits a
	# full filesystem). Remove the bulky preinstalled toolchains and
	# caches to give the linker headroom.
	run: \|
	sudo rm -rf /usr/share/dotnet
	sudo rm -rf /opt/ghc
	sudo rm -rf /usr/local/share/boost
	sudo rm -rf /usr/local/lib/android
	sudo rm -rf /opt/hostedtoolcache/CodeQL
	sudo rm -rf /usr/local/share/powershell
	sudo rm -rf /usr/local/share/chromium
	sudo rm -rf /usr/local/lib/node_modules
	sudo rm -rf /opt/microsoft
	sudo rm -rf /opt/google
	sudo rm -rf "$AGENT_TOOLSDIRECTORY"
	sudo docker image prune -af \|\| true
	df -h
	- uses: Swatinem/rust-cache@v2
	with:
	# liteparse-pdfium-sys's build.rs downloads a prebuilt PDFium
	# native library to ~/.cache/pdfium-rs (see docs/documents.md) —
	# outside the paths rust-cache tracks by default (~/.cargo/*,
	# target/). On a cache hit, cargo's build-script fingerprinting
	# sees the restored target/ output as still valid and skips
	# re-running build.rs, so it never re-downloads PDFium; the
	# runner then has no libpdfium.so and every documents-feature
	# test that touches PDFium panics. Caching this directory too
	# keeps the download in sync with the rest of the restored cache.
	cache-directories: ~/.cache/pdfium-rs
	- name: Check code format
	run: cargo fmt --all -- --check
	- name: Check the package for errors
	run: cargo check --all
	- name: Execute rust tests
	run: cargo llvm-cov --no-report nextest --all-features

	- name: Install integration test dependencies
	run: \|
	wget -qO- https://www.mongodb.org/static/pgp/server-7.0.asc \| sudo gpg --dearmor -o /usr/share/keyrings/mongodb-server-7.0.gpg
	echo "deb [ signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" \| sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list
	sudo apt-get update
	sudo apt-get install -y mongodb-mongosh redis-tools

	- name: Seed MySQL data
	run: \|
	mysql -h 127.0.0.1 -u skardi_user -pskardi_pass mydb <<'EOF'
	CREATE TABLE users (
	id INT PRIMARY KEY AUTO_INCREMENT,
	name VARCHAR(100) NOT NULL,
	email VARCHAR(100) UNIQUE NOT NULL
	);
	CREATE TABLE orders (
	id INT PRIMARY KEY AUTO_INCREMENT,
	user_id INT NOT NULL,
	product VARCHAR(100) NOT NULL,
	amount DECIMAL(10, 2) NOT NULL
	);
	CREATE TABLE user_order_stats (
	id INT PRIMARY KEY AUTO_INCREMENT,
	user_id INT NOT NULL,
	user_name VARCHAR(100) NOT NULL,
	user_email VARCHAR(100) NOT NULL,
	total_orders INT NOT NULL,
	total_spent DECIMAL(10, 2) NOT NULL,
	last_order_date VARCHAR(50),
	UNIQUE KEY unique_user (user_id)
	);
	INSERT INTO users (name, email) VALUES
	('Alice Smith', 'alice@example.com'),
	('Bob Johnson', 'bob@example.com'),
	('Carol Williams', 'carol@example.com');
	INSERT INTO orders (user_id, product, amount) VALUES
	(1, 'Laptop', 999.99),
	(2, 'Keyboard', 79.99),
	(3, 'Monitor', 299.99);
	EOF

	- name: Seed PostgreSQL data
	run: \|
	PGPASSWORD=skardi_pass psql -h 127.0.0.1 -U skardi_user -d mydb <<'EOF'
	CREATE EXTENSION IF NOT EXISTS vector;
	CREATE TABLE users (
	id SERIAL PRIMARY KEY,
	name VARCHAR(100) NOT NULL,
	email VARCHAR(100) UNIQUE NOT NULL
	);
	CREATE TABLE orders (
	id SERIAL PRIMARY KEY,
	user_id INT NOT NULL,
	product VARCHAR(100) NOT NULL,
	amount DECIMAL(10, 2) NOT NULL
	);
	CREATE TABLE user_order_stats (
	user_id INT PRIMARY KEY,
	user_name VARCHAR(100),
	user_email VARCHAR(100),
	total_orders INT,
	total_spent DECIMAL(10, 2),
	last_order_date VARCHAR(50)
	);
	INSERT INTO users (name, email) VALUES
	('Alice Smith', 'alice@example.com'),
	('Bob Johnson', 'bob@example.com'),
	('Carol Williams', 'carol@example.com');
	INSERT INTO orders (user_id, product, amount) VALUES
	(1, 'Laptop', 999.99),
	(2, 'Keyboard', 79.99),
	(3, 'Monitor', 299.99);
	CREATE TABLE articles (
	id SERIAL PRIMARY KEY,
	title VARCHAR(200) NOT NULL,
	body TEXT NOT NULL,
	category VARCHAR(50) NOT NULL
	);
	INSERT INTO articles (title, body, category) VALUES
	('Intro to Machine Learning', 'machine learning model training deep neural network supervised algorithms', 'ai'),
	('Natural Language Processing', 'natural language processing text classification sentiment analysis tokenization', 'ai'),
	('Database Query Optimization', 'database query optimization indexing performance tuning relational algebra', 'database'),
	('Deep Learning Advances', 'machine learning classification supervised training model convolutional neural network', 'research'),
	('Neural Network Architectures', 'deep learning neural network convolutional image recognition transformer attention', 'ai');
	-- pgvector-backed `docs` table — exercises the multi-row VALUES
	-- renderer's nested-array cell shape (`{"rows": [["a", "b", [v1,
	-- v2, v3, v4]], ...]}`) end-to-end. Mirrors the SeekDB `docs`
	-- table schema so the same parameter shape works on both.
	CREATE TABLE docs (
	id TEXT PRIMARY KEY,
	title TEXT NOT NULL,
	category TEXT NOT NULL,
	embedding vector(4) NOT NULL
	);
	EOF

	- name: Seed MongoDB data
	run: \|
	mongosh "mongodb://root:rootpass@127.0.0.1:27017/?authSource=admin" <<'EOF'
	use mydb

	db.createCollection("products", {
	validator: {
	$jsonSchema: {
	bsonType: "object",
	required: ["product_id", "name", "price"],
	properties: {
	product_id: { bsonType: "string" },
	name: { bsonType: "string" },
	category: { bsonType: "string" },
	price: { bsonType: "double" },
	in_stock: { bsonType: "bool" }
	}
	}
	}
	})

	db.products.insertMany([
	{ _id: "PROD001", product_id: "PROD001", name: "Laptop", category: "Electronics", price: 999.99, in_stock: true },
	{ _id: "PROD002", product_id: "PROD002", name: "Keyboard", category: "Electronics", price: 79.99, in_stock: true },
	{ _id: "PROD003", product_id: "PROD003", name: "Monitor", category: "Electronics", price: 299.99, in_stock: false },
	{ _id: "PROD004", product_id: "PROD004", name: "Mouse", category: "Electronics", price: 29.99, in_stock: true },
	{ _id: "PROD005", product_id: "PROD005", name: "Desk Chair", category: "Furniture", price: 199.99, in_stock: true }
	])

	db.createCollection("product_stats", {
	validator: {
	$jsonSchema: {
	bsonType: "object",
	required: ["stat_id"],
	properties: {
	stat_id: { bsonType: "string" },
	category: { bsonType: "string" },
	total_products: { bsonType: "long" },
	total_value: { bsonType: "double" },
	avg_price: { bsonType: "double" }
	}
	}
	}
	})

	// ─── Full-text search test collection ─────────────────────────
	db.createCollection("dataset_data_texts")
	db.dataset_data_texts.createIndex(
	{ fullTextToken: "text" },
	{ default_language: "none" }
	)
	db.dataset_data_texts.insertMany([
	{ _id: "data001", dataId: "data001", teamId: "team1", datasetId: "ds1", collectionId: "col1", fullTextToken: "machine learning model training deep neural network" },
	{ _id: "data002", dataId: "data002", teamId: "team1", datasetId: "ds1", collectionId: "col1", fullTextToken: "natural language processing text classification sentiment" },
	{ _id: "data003", dataId: "data003", teamId: "team1", datasetId: "ds1", collectionId: "col2", fullTextToken: "database query optimization indexing performance tuning" },
	{ _id: "data004", dataId: "data004", teamId: "team2", datasetId: "ds2", collectionId: "col3", fullTextToken: "machine learning classification supervised training model" },
	{ _id: "data005", dataId: "data005", teamId: "team1", datasetId: "ds1", collectionId: "col1", fullTextToken: "deep learning neural network convolutional image recognition" }
	])
	EOF

	- name: Seed Redis data
	run: \|
	redis-cli -h 127.0.0.1 <<'EOF'
	HSET mydb:products:PROD001 name "Laptop" category "Electronics" price "999.99" in_stock "true"
	HSET mydb:products:PROD002 name "Keyboard" category "Electronics" price "79.99" in_stock "true"
	HSET mydb:products:PROD003 name "Monitor" category "Electronics" price "299.99" in_stock "false"
	HSET mydb:products:PROD004 name "Mouse" category "Electronics" price "29.99" in_stock "true"
	HSET mydb:products:PROD005 name "Desk Chair" category "Furniture" price "199.99" in_stock "true"
	EOF

	- name: Wait for SeekDB to accept connections
	run: \|
	for i in $(seq 1 60); do
	if mysql -h 127.0.0.1 -P 2881 -u "root@sys" -e "SELECT 1" >/dev/null 2>&1; then
	echo "SeekDB is ready"
	break
	fi
	echo "Waiting for SeekDB... (attempt $i/60)"
	sleep 5
	done

	- name: Seed SeekDB data
	run: \|
	mysql -h 127.0.0.1 -P 2881 -u "root@sys" <<'EOF'
	CREATE DATABASE IF NOT EXISTS mydb;
	USE mydb;
	CREATE TABLE IF NOT EXISTS users (
	id INT PRIMARY KEY AUTO_INCREMENT,
	name VARCHAR(100) NOT NULL,
	email VARCHAR(100) UNIQUE NOT NULL
	);
	CREATE TABLE IF NOT EXISTS orders (
	id INT PRIMARY KEY AUTO_INCREMENT,
	user_id INT NOT NULL,
	product VARCHAR(100) NOT NULL,
	amount DECIMAL(10, 2) NOT NULL
	);
	CREATE TABLE IF NOT EXISTS articles (
	id INT PRIMARY KEY AUTO_INCREMENT,
	title VARCHAR(200) NOT NULL,
	body TEXT NOT NULL,
	category VARCHAR(50) NOT NULL,
	FULLTEXT INDEX ft_body (body) WITH PARSER IK
	);
	CREATE TABLE IF NOT EXISTS docs (
	id INT PRIMARY KEY AUTO_INCREMENT,
	title VARCHAR(200) NOT NULL,
	category VARCHAR(50) NOT NULL,
	embedding VECTOR(4),
	VECTOR INDEX idx_embedding (embedding) WITH (TYPE = HNSW, DISTANCE = L2)
	);
	-- Inner-product variant to verify NEGATIVE_INNER_PRODUCT() + HNSW
	-- actually work end-to-end; L2/COSINE are exercised via the docs table.
	CREATE TABLE IF NOT EXISTS docs_ip (
	id INT PRIMARY KEY AUTO_INCREMENT,
	title VARCHAR(200) NOT NULL,
	embedding VECTOR(4),
	VECTOR INDEX idx_embedding_ip (embedding) WITH (TYPE = HNSW, DISTANCE = INNER_PRODUCT)
	);
	INSERT INTO users (name, email) VALUES
	('Alice Smith', 'alice@example.com'),
	('Bob Johnson', 'bob@example.com'),
	('Carol Williams', 'carol@example.com');
	INSERT INTO orders (user_id, product, amount) VALUES
	(1, 'Laptop', 999.99),
	(2, 'Keyboard', 79.99),
	(3, 'Monitor', 299.99);
	INSERT INTO articles (title, body, category) VALUES
	('Intro to Machine Learning', 'machine learning model training deep neural network supervised algorithms', 'ai'),
	('Natural Language Processing', 'natural language processing text classification sentiment analysis', 'ai'),
	('Database Query Optimization', 'database query optimization indexing performance tuning relational', 'database'),
	('Deep Learning Advances', 'machine learning classification supervised training model convolutional neural', 'research'),
	('Neural Network Architectures', 'deep learning neural network convolutional image recognition transformer', 'ai');
	INSERT INTO docs (title, category, embedding) VALUES
	('doc-a', 'electronics', '[1.0, 0.0, 0.0, 0.0]'),
	('doc-b', 'electronics', '[0.0, 1.0, 0.0, 0.0]'),
	('doc-c', 'books', '[0.0, 0.0, 1.0, 0.0]'),
	('doc-d', 'electronics', '[1.0, 1.0, 0.0, 0.0]'),
	('doc-e', 'books', '[0.5, 0.5, 0.5, 0.5]');
	INSERT INTO docs_ip (title, embedding) VALUES
	('ip-a', '[1.0, 0.0, 0.0, 0.0]'),
	('ip-b', '[0.0, 1.0, 0.0, 0.0]'),
	('ip-c', '[1.0, 1.0, 0.0, 0.0]'),
	('ip-d', '[0.5, 0.5, 0.5, 0.5]');
	EOF

	- name: Start and seed InfluxDB 3
	# InfluxDB 3 Core needs `influxdb3 serve <args>`, which a GitHub
	# Actions service container can't express (no command override), so
	# run it as a plain container. `--object-store memory` keeps it
	# ephemeral and disk-free; `--without-auth` skips token management for
	# the test fixture.
	run: \|
	docker run -d --name influxdb3 -p 8181:8181 \
	influxdb:3-core influxdb3 serve \
	--node-id ci-node0 \
	--object-store memory \
	--without-auth
	echo "Waiting for InfluxDB 3 to become healthy..."
	for i in $(seq 1 60); do
	if curl -fsS http://127.0.0.1:8181/health >/dev/null 2>&1; then
	echo "InfluxDB 3 is ready"
	break
	fi
	echo "Waiting for InfluxDB 3... (attempt $i/60)"
	sleep 2
	done
	curl -fsS -XPOST "http://127.0.0.1:8181/api/v3/configure/database" \
	-H "Content-Type: application/json" -d '{"db": "metrics"}'
	curl -fsS "http://127.0.0.1:8181/api/v3/write_lp?db=metrics&precision=second" \
	--data-binary @- <<'EOF'
	cpu,host=host1,region=us-west usage_user=12.5,usage_system=3.2 1700000000
	cpu,host=host1,region=us-west usage_user=64.1,usage_system=9.8 1700000060
	cpu,host=host2,region=us-west usage_user=41.0,usage_system=6.0 1700000000
	cpu,host=host2,region=us-west usage_user=88.7,usage_system=12.3 1700000060
	cpu,host=host3,region=us-east usage_user=22.4,usage_system=4.1 1700000000
	mem,host=host1,region=us-west used_percent=48.2 1700000000
	mem,host=host2,region=us-west used_percent=73.9 1700000000
	mem,host=host3,region=us-east used_percent=31.5 1700000000
	EOF

	- name: Seed ClickHouse data
	# The HTTP interface accepts one statement per request, so the seed
	# script is split on blank lines and posted statement by statement.
	run: \|
	seed() {
	curl -fsS "http://127.0.0.1:8123/?user=skardi_user&password=skardi_pass" \
	--data-binary "$1" > /dev/null
	}
	seed "CREATE TABLE mydb.users (
	id UInt32,
	name String,
	email String
	) ENGINE = MergeTree ORDER BY id"
	seed "INSERT INTO mydb.users VALUES
	(1, 'Alice Smith', 'alice@example.com'),
	(2, 'Bob Johnson', 'bob@example.com'),
	(3, 'Carol Williams', 'carol@example.com')"
	seed "CREATE TABLE mydb.orders (
	id UInt32,
	user_id UInt32,
	product String,
	amount Float64
	) ENGINE = MergeTree ORDER BY id"
	seed "INSERT INTO mydb.orders VALUES
	(1, 1, 'Laptop', 999.99),
	(2, 2, 'Keyboard', 79.99),
	(3, 3, 'Monitor', 299.99)"
	seed "CREATE TABLE mydb.products (
	product_id String,
	name String,
	category Nullable(String),
	price Float64,
	in_stock Bool
	) ENGINE = MergeTree ORDER BY product_id"
	# PROD005 has a NULL category on purpose — exercises NULL handling.
	seed "INSERT INTO mydb.products VALUES
	('PROD001', 'Laptop', 'Electronics', 999.99, true),
	('PROD002', 'Keyboard', 'Electronics', 79.99, true),
	('PROD003', 'Monitor', 'Electronics', 299.99, false),
	('PROD004', 'Mouse', 'Electronics', 29.99, true),
	('PROD005', 'Desk Chair', NULL, 199.99, true)"
	# Deliberately left empty — exercises schema inference on empty tables.
	seed "CREATE TABLE mydb.empty_metrics (
	ts DateTime,
	value Float64
	) ENGINE = MergeTree ORDER BY ts"

	- name: Seed DynamoDB data
	env:
	EP: http://127.0.0.1:8000
	run: \|
	# Wait for DynamoDB Local to accept connections.
	for i in $(seq 1 30); do
	if aws dynamodb list-tables --endpoint-url "$EP" >/dev/null 2>&1; then
	echo "DynamoDB Local is ready"
	break
	fi
	echo "Waiting for DynamoDB Local... (attempt $i/30)"
	sleep 2
	done

	aws dynamodb create-table --endpoint-url "$EP" \
	--table-name products \
	--attribute-definitions AttributeName=product_id,AttributeType=S \
	--key-schema AttributeName=product_id,KeyType=HASH \
	--billing-mode PAY_PER_REQUEST
	aws dynamodb wait table-exists --endpoint-url "$EP" --table-name products

	put() { aws dynamodb put-item --endpoint-url "$EP" --table-name products --item "$1"; }
	put '{"product_id":{"S":"PROD001"},"name":{"S":"Laptop"},"category":{"S":"Electronics"},"price":{"N":"999.99"},"in_stock":{"BOOL":true}}'
	put '{"product_id":{"S":"PROD002"},"name":{"S":"Keyboard"},"category":{"S":"Electronics"},"price":{"N":"79.99"},"in_stock":{"BOOL":true}}'
	put '{"product_id":{"S":"PROD003"},"name":{"S":"Monitor"},"category":{"S":"Electronics"},"price":{"N":"299.99"},"in_stock":{"BOOL":false}}'
	put '{"product_id":{"S":"PROD004"},"name":{"S":"Mouse"},"category":{"S":"Electronics"},"price":{"N":"29.99"},"in_stock":{"BOOL":true}}'
	# NULL-bearing row: no `category` attribute, exercising NULL handling.
	put '{"product_id":{"S":"PROD005"},"name":{"S":"Desk Chair"},"price":{"N":"199.99"},"in_stock":{"BOOL":true}}'

	- name: Start skardi-server for CLI e2e smoke tests
	run: \|
	# The CLI's #[ignore] e2e_smoke tests are integration tests whose
	# backing service is skardi-server itself; they hit the default
	# http://127.0.0.1:8080 (no ctx/pipelines needed). The nextest step
	# above already compiled the server binary into the llvm-cov
	# target dir, so this starts without rebuilding.
	./target/llvm-cov-target/debug/skardi-server --port 8080 &
	for i in $(seq 1 30); do
	curl -sf http://127.0.0.1:8080/health >/dev/null && exit 0
	sleep 1
	done
	echo "skardi-server failed to become healthy" >&2
	exit 1

	- name: Execute Integration tests
	run: cargo llvm-cov --no-report nextest --all-features -- --ignored

	# MinIO gives the `documents` connector's object-store path real S3
	# coverage without needing an AWS account or credentials in CI. Started
	# with `docker run` rather than a `services:` entry because service
	# containers cannot override the image command, and the MinIO image needs
	# `server /data`.
	- name: Start MinIO for documents S3 tests
	run: \|
	docker run -d --name skardi-minio \
	-p 127.0.0.1:9000:9000 \
	-e MINIO_ROOT_USER=skardiminio \
	-e MINIO_ROOT_PASSWORD=skardiminio123 \
	quay.io/minio/minio server /data
	# The image has no shell tooling for a container healthcheck, so poll
	# readiness from the runner (same approach as DynamoDB Local above).
	for i in $(seq 1 30); do
	if curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null 2>&1; then
	echo "minio ready after ${i}s"
	break
	fi
	if [ "$i" = "30" ]; then
	echo "minio failed to become ready"
	docker logs skardi-minio
	exit 1
	fi
	sleep 1
	done
	AWS_ACCESS_KEY_ID=skardiminio AWS_SECRET_ACCESS_KEY=skardiminio123 \
	aws --endpoint-url http://127.0.0.1:9000 s3 mb s3://skardi-ci-documents

	# These are `#[ignore]`d and skip themselves unless DOCUMENTS_S3_LIVE=1, so
	# the step above's credentials are what actually arms them. Set at step
	# scope, not job scope: the job-level AWS_* values are the `dummy`
	# placeholders DynamoDB Local expects, and MinIO rejects a root password
	# shorter than 8 characters.
	- name: Execute documents S3 live tests against MinIO
	env:
	DOCUMENTS_S3_LIVE: "1"
	DOCUMENTS_S3_BUCKET: skardi-ci-documents
	# `AmazonS3Builder::from_env()` honours both of these, so the real S3
	# HTTP client path is exercised, not a stub.
	AWS_ENDPOINT: http://127.0.0.1:9000
	AWS_ALLOW_HTTP: "true"
	AWS_REGION: us-east-1
	AWS_ACCESS_KEY_ID: skardiminio
	AWS_SECRET_ACCESS_KEY: skardiminio123
	# `--no-tests=fail` is explicit rather than relying on the current
	# default: if these tests are ever renamed out of the filter's reach, the
	# step must fail loudly instead of reporting success having run nothing.
	run: \|
	cargo llvm-cov --no-report nextest --all-features \
	-E 'test(/live_s3_/)' --no-tests=fail -- --ignored

	- name: Stop MinIO
	if: always()
	run: docker rm -f skardi-minio \|\| true

	- name: Generate coverage report (lcov)
	run: cargo llvm-cov report --lcov --output-path lcov.info

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v5
	with:
	files: lcov.info
	fail_ci_if_error: false
	token: ${{ secrets.CODECOV_TOKEN }}

Navigation Menu

Uh oh!

feat(sources): Gmail source pack (milestone 5.4) #758

Workflow file

feat(sources): Gmail source pack (milestone 5.4) #758

Uh oh!

Workflow file for this run