Skip to content

Commit aadb9c4

Browse files
timgittimgit
authored
Merge pull request #815 from timgit/feat/distributed
Alternative DBs
2 parents 1ac9463 + f9a2cd7 commit aadb9c4

50 files changed

Lines changed: 2539 additions & 203 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci-cockroachdb.yml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
on:
2+
push:
3+
branches: [master]
4+
tags-ignore: ['**']
5+
pull_request:
6+
branches: [master]
7+
# Allow the full-suite compatibility run to be triggered by hand...
8+
workflow_dispatch:
9+
# ...and on a nightly schedule, so regressions in CockroachDB support surface
10+
# without waiting for someone to run it manually.
11+
schedule:
12+
- cron: '0 6 * * *'
13+
14+
name: CI (CockroachDB)
15+
16+
jobs:
17+
# Fast feedback on every push/PR: the distributed-mode invariants that the rest of the
18+
# suite can't express, run against a real CockroachDB cluster.
19+
test:
20+
runs-on: ubuntu-latest
21+
22+
steps:
23+
- name: Checkout code
24+
uses: actions/checkout@v4
25+
26+
- name: Set up Node.js
27+
uses: actions/setup-node@v4
28+
with:
29+
node-version: 22
30+
31+
- name: Start CockroachDB cluster
32+
run: |
33+
docker compose -f docker-compose.cockroach.yaml up -d
34+
# `up --wait` can't be used here: the one-shot init/setup containers exit by design, which
35+
# newer compose reports as a failed service. Block on the setup container instead - it
36+
# creates the pgboss database once the cluster is initialized.
37+
docker wait pgboss-cockroach-cockroachdb-setup-1 || true
38+
docker compose -f docker-compose.cockroach.yaml exec -T cockroachdb-1 cockroach sql --insecure -e "SHOW DATABASES" | grep -q pgboss
39+
40+
- name: Install
41+
run: npm ci
42+
43+
- name: Test distributed database mode
44+
run: npm run test:cockroachdb
45+
46+
# Full suite against CockroachDB. This is the compatibility matrix / regression signal: it is
47+
# slow (CockroachDB pays ~8s of online-DDL per test, and the suite rebuilds the schema per test),
48+
# so it does not gate pushes/PRs. It runs nightly and on demand. PostgreSQL-only tests skip
49+
# automatically via testHelper's itPostgresOnly/describePostgresOnly.
50+
test-full-suite:
51+
if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
52+
runs-on: ubuntu-latest
53+
timeout-minutes: 120
54+
55+
steps:
56+
- name: Checkout code
57+
uses: actions/checkout@v4
58+
59+
- name: Set up Node.js
60+
uses: actions/setup-node@v4
61+
with:
62+
node-version: 22
63+
64+
- name: Start CockroachDB cluster
65+
run: |
66+
docker compose -f docker-compose.cockroach.yaml up -d
67+
# See the `test` job: block on the setup container (which creates the pgboss database)
68+
# rather than `up --wait`, which mishandles the one-shot init/setup containers.
69+
docker wait pgboss-cockroach-cockroachdb-setup-1 || true
70+
docker compose -f docker-compose.cockroach.yaml exec -T cockroachdb-1 cockroach sql --insecure -e "SHOW DATABASES" | grep -q pgboss
71+
72+
- name: Install
73+
run: npm ci
74+
75+
- name: Test full suite (CockroachDB)
76+
run: npm run test:cockroachdb:full

‎.github/workflows/ci.yml‎

Lines changed: 71 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,20 +45,87 @@ jobs:
4545
run: npm run cover
4646
env:
4747
POSTGRES_HOST: postgres
48-
48+
4949
- name: Coveralls
5050
uses: coverallsapp/github-action@v2
5151
with:
52-
flag-name: run-${{ join(matrix.*, '-') }}
52+
flag-name: standard
5353
parallel: true
5454

55+
# Runs the full suite in distributed database mode on Postgres. distributedDatabaseMode is a pure
56+
# runtime fetch-strategy toggle, so every test (existing and new) is automatically exercised
57+
# against the distributed code paths here - fast and reliable, without CockroachDB's slow DDL.
58+
test-distributed:
59+
runs-on: ubuntu-latest
60+
container: node:24
61+
services:
62+
postgres:
63+
image: postgres
64+
env:
65+
POSTGRES_PASSWORD: postgres
66+
options: >-
67+
--health-cmd pg_isready
68+
--health-interval 10s
69+
--health-timeout 5s
70+
--health-retries 5
71+
72+
steps:
73+
- name: Checkout code
74+
uses: actions/checkout@v4
75+
76+
- name: Set up Node.js
77+
uses: actions/setup-node@v4
78+
with:
79+
node-version: 24
80+
81+
- name: Install
82+
run: npm install
83+
84+
# Collect coverage here too: the distributed fetch strategy exercises code paths
85+
# (completeDistributed/failDistributed, distributed branches in plans.ts) that the
86+
# standard `test` job never hits. Uploading as a second parallel flag makes Coveralls
87+
# report the union, so those paths are measured instead of showing as uncovered.
88+
- name: Test (distributed mode)
89+
run: npm run cover
90+
env:
91+
POSTGRES_HOST: postgres
92+
DISTRIBUTED: 'true'
93+
94+
- name: Coveralls
95+
uses: coverallsapp/github-action@v2
96+
with:
97+
flag-name: distributed
98+
parallel: true
99+
100+
# Runs the full suite against embedded PGlite (in-process WASM PostgreSQL). No service container is
101+
# needed — the database runs inside the test process. Connection-model-specific tests (custom pg
102+
# connections, pool internals, multi-master, distributed mode) are skipped via itPglite/describePglite.
103+
test-pglite:
104+
runs-on: ubuntu-latest
105+
container: node:24
106+
107+
steps:
108+
- name: Checkout code
109+
uses: actions/checkout@v4
110+
111+
- name: Set up Node.js
112+
uses: actions/setup-node@v4
113+
with:
114+
node-version: 24
115+
116+
- name: Install
117+
run: npm install
118+
119+
- name: Test (pglite)
120+
run: npm run test:pglite
121+
55122
coverage:
56-
needs: test
123+
needs: [test, test-distributed]
57124
if: ${{ always() }}
58125
runs-on: ubuntu-latest
59126
steps:
60127
- name: Coveralls Finished
61128
uses: coverallsapp/github-action@v2
62129
with:
63130
parallel-finished: true
64-
carryforward: "run-1,run-2"
131+
carryforward: "standard,distributed"

‎README.md‎

Lines changed: 2 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ This will likely cater the most to teams already familiar with the simplicity of
4242

4343
## Summary
4444
* Exactly-once job delivery
45-
* Create jobs within your existing database transaction
45+
* Create jobs in an existing db transaction, including adapters for popular ORMs such as Drizzle, Knex, Kysely, Prisma
4646
* Backpressure-compatible polling workers
4747
* Cron scheduling
4848
* Queue storage policies to support a variety of rate limiting, debouncing, and concurrency use cases
@@ -51,6 +51,7 @@ This will likely cater the most to teams already familiar with the simplicity of
5151
* SQL support for non-Node.js runtimes for most operations
5252
* Serverless function compatible
5353
* Multi-master compatible (for example, in a Kubernetes ReplicaSet)
54+
* [Additional database backends](https://timgit.github.io/pg-boss/docs/database-backends) for Postgres-based databases such as CockroachDB, YugabyteDB and Citus. Or, use embedded PGlite for running entirely in-process.
5455

5556
## CLI
5657

@@ -70,48 +71,6 @@ A HTTP proxy is available in the [`@pg-boss/proxy`](https://www.npmjs.com/packag
7071

7172
See the [proxy documentation](https://github.com/timgit/pg-boss/blob/master/packages/proxy/README.md) for full configuration and deployment options.
7273

73-
## ORM Transaction Adapters
74-
75-
pg-boss ships adapters for running operations inside ORM-managed transactions. Each adapter wraps the ORM's transaction object as an `IDatabase` you can pass via the `db` option on `send()`, `insert()`, `fetch()`, `complete()`, and other methods.
76-
77-
### Knex / Kysely / Prisma
78-
79-
```ts
80-
import { fromKnex, fromKysely, fromPrisma } from 'pg-boss'
81-
```
82-
83-
```ts
84-
// Knex
85-
await knex.transaction(async (trx) => {
86-
await boss.send('my-queue', data, { db: fromKnex(trx) })
87-
})
88-
89-
// Kysely
90-
await db.transaction().execute(async (trx) => {
91-
await boss.send('my-queue', data, { db: fromKysely(trx) })
92-
})
93-
94-
// Prisma (v7+ with @prisma/adapter-pg)
95-
await prisma.$transaction(async (tx) => {
96-
await boss.send('my-queue', data, { db: fromPrisma(tx) })
97-
})
98-
```
99-
100-
### Drizzle
101-
102-
The Drizzle adapter accepts the `sql` tagged-template function from `drizzle-orm` as a second argument so it can construct parameterised queries without a runtime dependency on `drizzle-orm`.
103-
104-
```ts
105-
import { fromDrizzle } from 'pg-boss'
106-
import { sql } from 'drizzle-orm'
107-
```
108-
109-
```ts
110-
await db.transaction(async (tx) => {
111-
await boss.send('my-queue', data, { db: fromDrizzle(tx, sql) })
112-
})
113-
```
114-
11574
## Requirements
11675
* Node 22.12 or higher for CommonJS's require(esm)
11776
* PostgreSQL 13 or higher

‎docker-compose.citus.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Single-node Citus coordinator (the Citus extension on PostgreSQL). Listens on 5432 in-container,
2+
# mapped to 5434 to avoid clashing with the default `db`. pg-boss never calls
3+
# create_distributed_table(), so its tables stay local to the coordinator and behave like plain
4+
# PostgreSQL — this exercises the standard path with Citus loaded. Its own compose project so it
5+
# never starts alongside the default Postgres (docker-compose.yaml).
6+
# Usage: docker compose -f docker-compose.citus.yaml up -d
7+
name: pgboss-citus
8+
9+
services:
10+
citus:
11+
image: citusdata/citus:latest
12+
ports:
13+
- 5434:5432
14+
environment:
15+
- POSTGRES_PASSWORD=postgres
16+
healthcheck:
17+
test: ["CMD-SHELL", "pg_isready -U postgres"]
18+
interval: 5s
19+
timeout: 5s
20+
retries: 30
21+
22+
citus-setup:
23+
image: citusdata/citus:latest
24+
depends_on:
25+
citus:
26+
condition: service_healthy
27+
environment:
28+
- PGPASSWORD=postgres
29+
entrypoint: ["/bin/sh", "-c"]
30+
# PostgreSQL has no CREATE DATABASE IF NOT EXISTS, so ignore the "already exists" error, then
31+
# ensure the citus extension is present in the pgboss database.
32+
command: ["psql -h citus -U postgres -c 'CREATE DATABASE pgboss' || true; psql -h citus -U postgres -d pgboss -c 'CREATE EXTENSION IF NOT EXISTS citus'"]
33+
restart: "no"

‎docker-compose.cockroach.yaml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Three-node CockroachDB cluster for the compatibility suite. Its own compose project so it never
2+
# starts alongside the default Postgres (docker-compose.yaml).
3+
# Usage: docker compose -f docker-compose.cockroach.yaml up -d --wait
4+
name: pgboss-cockroach
5+
6+
# The three nodes are identical except for their advertise address, ports, and data volume.
7+
x-cockroach-node: &cockroach-node
8+
image: cockroachdb/cockroach:latest
9+
healthcheck:
10+
test: ["CMD", "curl", "-sf", "http://localhost:8080/health"]
11+
interval: 2s
12+
timeout: 5s
13+
retries: 30
14+
start_period: 10s
15+
16+
services:
17+
cockroachdb-1:
18+
<<: *cockroach-node
19+
ports:
20+
- 26257:26257
21+
- 8080:8080
22+
volumes:
23+
- cockroach_volume_1:/cockroach/cockroach-data
24+
command: start --insecure --advertise-addr=cockroachdb-1 --join=cockroachdb-1,cockroachdb-2,cockroachdb-3
25+
26+
cockroachdb-2:
27+
<<: *cockroach-node
28+
volumes:
29+
- cockroach_volume_2:/cockroach/cockroach-data
30+
command: start --insecure --advertise-addr=cockroachdb-2 --join=cockroachdb-1,cockroachdb-2,cockroachdb-3
31+
32+
cockroachdb-3:
33+
<<: *cockroach-node
34+
volumes:
35+
- cockroach_volume_3:/cockroach/cockroach-data
36+
command: start --insecure --advertise-addr=cockroachdb-3 --join=cockroachdb-1,cockroachdb-2,cockroachdb-3
37+
38+
cockroachdb-init:
39+
image: cockroachdb/cockroach:latest
40+
depends_on:
41+
cockroachdb-1:
42+
condition: service_healthy
43+
cockroachdb-2:
44+
condition: service_healthy
45+
cockroachdb-3:
46+
condition: service_healthy
47+
entrypoint: ["/bin/sh", "-c"]
48+
command: ["cockroach init --insecure --host=cockroachdb-1 || true"]
49+
restart: "no"
50+
51+
cockroachdb-setup:
52+
image: cockroachdb/cockroach:latest
53+
depends_on:
54+
cockroachdb-init:
55+
condition: service_completed_successfully
56+
command: sql --insecure --host=cockroachdb-1 -e "CREATE DATABASE IF NOT EXISTS pgboss"
57+
restart: "no"
58+
59+
volumes:
60+
cockroach_volume_1:
61+
cockroach_volume_2:
62+
cockroach_volume_3:

‎docker-compose.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
# Default single-node Postgres for the standard test suite.
2+
# Usage: docker compose up -d
3+
# The alternative backends each have their own file (docker-compose.cockroach.yaml,
4+
# docker-compose.yugabyte.yaml, docker-compose.citus.yaml) so they never start alongside this default.
15
services:
26
db:
37
image: postgres:18

‎docker-compose.yugabyte.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Single-node YugabyteDB (PostgreSQL-compatible) for the compatibility suite. YSQL listens on 5433.
2+
# Its own compose project so it never starts alongside the default Postgres (docker-compose.yaml).
3+
# Usage: docker compose -f docker-compose.yugabyte.yaml up -d
4+
name: pgboss-yugabyte
5+
6+
services:
7+
yugabytedb:
8+
image: yugabytedb/yugabyte:latest
9+
ports:
10+
- 5433:5433
11+
- 15433:15433
12+
volumes:
13+
- yugabyte_volume:/home/yugabyte/var
14+
command: bin/yugabyted start --background=false
15+
healthcheck:
16+
# YSQL binds to the container's advertised address, so connect via the container hostname.
17+
test: ["CMD-SHELL", "bin/ysqlsh -h $$(hostname) -U yugabyte -d yugabyte -c 'select 1' || exit 1"]
18+
interval: 5s
19+
timeout: 10s
20+
retries: 40
21+
start_period: 30s
22+
23+
yugabytedb-setup:
24+
image: yugabytedb/yugabyte:latest
25+
depends_on:
26+
yugabytedb:
27+
condition: service_healthy
28+
entrypoint: ["/bin/sh", "-c"]
29+
# YugabyteDB (PostgreSQL 15) has no CREATE DATABASE IF NOT EXISTS, so ignore the "already exists" error.
30+
command: ["bin/ysqlsh -h yugabytedb -U yugabyte -d yugabyte -c 'CREATE DATABASE pgboss' || true"]
31+
restart: "no"
32+
33+
volumes:
34+
yugabyte_volume:

‎docs/.vitepress/config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ export default defineConfig({
4444
{ text: 'CLI', link: '/cli' },
4545
{ text: 'Dashboard', link: '/dashboard' },
4646
{ text: 'Proxy', link: '/proxy' },
47+
{ text: 'Database Backends', link: '/database-backends' },
4748
{
4849
text: 'API',
4950
collapsed: false,

0 commit comments

Comments
 (0)