Replace sqlite with postgres

.dev.vars.example

@@ -19,3 +19,9 @@ ALLOW_LOCAL_AUTH="true"
 # - VITE_CLIENT_ID: Your OAuth client ID
 #
 # The backend expects RS256 JWT tokens from the OIDC provider.
+
+# Postgres
+# Direct PostgreSQL connection
+# PG_CONNECTION_STRING="postgresql://postgres:password@localhost:5432/intheloop-dev?sslmode=disable"
+# PgBouncer connection (recommended for production-like testing)
+PG_CONNECTION_STRING="postgresql://postgres:password@localhost:6432/intheloop-dev?sslmode=disable"

DOCKER.md

@@ -25,13 +25,13 @@ docker compose build
 To build without cache:
 
 ```shell
-docker-compose build --no-cache
+docker compose build --no-cache
 ```
 
 To build and start all services:
 
 ```shell
-docker-compose up --build
+docker compose up --build
 ```
 
 ## Running All Images
@@ -67,7 +67,7 @@ Build the web image:
 docker build -f Dockerfile.web -t intheloop-web:latest .
 ```
 
-**Running with docker-compose (recommended):**
+**Running with docker compose (recommended):**
 
 The web service automatically connects to the `sync` service via Docker network DNS:
 
@@ -90,7 +90,7 @@ docker run -p 5173:5173 -e SYNC_HOST=172.17.0.1 intheloop-web:latest
 docker run -p 5173:5173 -e SYNC_HOST=<container-ip> intheloop-web:latest
 ```
 
-The web service will be available at `http://localhost:5173`. By default, `SYNC_HOST` is set to `sync` (for docker-compose usage).
+The web service will be available at `http://localhost:5173`. By default, `SYNC_HOST` is set to `sync` (for docker compose usage).
 
 #### Iframe Outputs Service
 
@@ -128,7 +128,7 @@ The sync service will be available at `http://localhost:8787`.
 
 ### Web Service
 
-- `SYNC_HOST` (default: `sync`): Hostname or IP address of the sync service. Used by nginx to proxy API requests. Set to `sync` for docker-compose, or override for standalone runs.
+- `SYNC_HOST` (default: `sync`): Hostname or IP address of the sync service. Used by nginx to proxy API requests. Set to `sync` for docker compose, or override for standalone runs.
 
 ### Sync Service
 
@@ -140,7 +140,7 @@ The sync service uses environment variables from `.dev.vars` file. See `.dev.var
 
 This error occurs when running the web container standalone without setting `SYNC_HOST`. Either:
 
-1. Use docker-compose to run all services together, or
+1. Use docker compose to run all services together, or
 2. Set `SYNC_HOST` environment variable when running the web container standalone
 
 ### Web service can't connect to sync service

Dockerfile.sync

@@ -14,6 +14,8 @@ COPY packages/agent-core/package.json ./packages/agent-core/
 COPY packages/ai-core/package.json ./packages/ai-core/
 COPY packages/pyodide-runtime/package.json ./packages/pyodide-runtime/
 COPY packages/schema/package.json ./packages/schema/
+COPY packages/livestore-postgres/package.json ./packages/livestore-postgres/
+COPY packages/livestore-postgres ./packages/livestore-postgres/
 
 # Install dependencies
 RUN pnpm install
@@ -25,8 +27,12 @@ COPY . .
 COPY .env.example .env
 COPY .dev.vars.example .dev.vars
 
+# Copy entrypoint script
+COPY scripts/docker-sync-entrypoint.sh /usr/local/bin/docker-sync-entrypoint.sh
+RUN chmod +x /usr/local/bin/docker-sync-entrypoint.sh
+
 # Expose Wrangler's default port
 EXPOSE 8787
 
-# Run migrations then start the sync service
-CMD pnpm run db:migrate && pnpm run docker:sync
+# Use entrypoint script to generate .dev.vars from environment variables
+ENTRYPOINT ["/usr/local/bin/docker-sync-entrypoint.sh"]

README.md

@@ -70,6 +70,30 @@ The example files contain working defaults for local development:
 - `.env.example` → `.env` - Frontend environment variables (Vite)
 - `.dev.vars.example` → `.dev.vars` - Backend environment variables (Worker)
 
+## Docker Services
+
+The `docker-compose.yml` includes several services:
+
+1. **web** - The main web application (port 5173)
+2. **iframe-outputs** - Sandboxed output rendering server (port 8000)
+3. **sync** - Backend sync server (port 8787)
+4. **PostgreSQL** - The database server (port 5432 by default)
+5. **PgBouncer** - Connection pooler for PostgreSQL (port 6432 by default)
+
+### Using PgBouncer
+
+To connect through PgBouncer instead of directly to PostgreSQL, update your connection string in `.dev.vars`:
+
+```toml
+# Direct PostgreSQL connection
+PG_CONNECTION_STRING = "postgresql://postgres:password@localhost:5432/intheloop-dev?sslmode=disable"
+
+# PgBouncer connection (recommended for production-like testing)
+PG_CONNECTION_STRING = "postgresql://postgres:password@localhost:6432/intheloop-dev?sslmode=disable"
+```
+
+PgBouncer uses transaction-level pooling, which helps manage connection limits and improves performance under load.
+
 ### 2. Create Your First Notebook
 
 1. Open http://localhost:5173

backend/sync.ts

@@ -1,13 +1,12 @@
-import {
-  makeDurableObject,
-  handleWebSocket,
-} from "@livestore/sync-cf/cf-worker";
+import { handleWebSocket } from "@livestore/sync-cf/cf-worker";
+import { makePostgres } from "@runtimed/livestore-postgres";
 import { type Env, type ExecutionContext } from "./types";
 
 import { getValidatedUser } from "./auth";
 import { Schema } from "@runtimed/schema";
 
-export class WebSocketServer extends makeDurableObject({
+export class WebSocketServer extends makePostgres({
+  // These are needed, even if they are empty
   onPush: async (message) => {
     console.log("onPush", message.batch);
   },

backend/types.ts

@@ -33,6 +33,8 @@ export type Env = {
   WEBSOCKET_SERVER: DurableObjectNamespace;
   DB: D1Database;
 
+  PG_CONNECTION_STRING: string;
+
   // Secrets
   AUTH_ISSUER: string;
 

docker-compose.yml

@@ -10,6 +10,9 @@ services:
       - "5173:5173"
     environment:
       ALLOW_LOCAL_AUTH: "true"
+    depends_on:
+      sync:
+        condition: service_started
     networks:
       - intheloop-network
 
@@ -30,6 +33,57 @@ services:
       dockerfile: Dockerfile.sync
     ports:
       - "8787:8787"
+    environment:
+      ALLOW_LOCAL_AUTH: "true"
+      PG_CONNECTION_STRING: "postgresql://postgres:password@postgres:5432/intheloop-dev?sslmode=disable"
+      AUTH_ISSUER: "http://localhost:8787/local_oidc"
+      DEPLOYMENT_ENV: "development"
+      SERVICE_PROVIDER: "local"
+      ADMIN_SECRET: "dev-admin-secret"
+      EXTENSION_CONFIG: "{}"
+      ARTIFACT_STORAGE: "local"
+      ARTIFACT_THRESHOLD: "16384"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    networks:
+      - intheloop-network
+
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_DB: intheloop-dev
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: password
+    ports:
+      - 5432:5432
+    tmpfs:
+      - /var/lib/postgresql/data
+      - /tmp
+    command:
+      - -c
+      - listen_addresses=*
+      - -c
+      - wal_level=logical
+    networks:
+      - intheloop-network
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  pgbouncer:
+    image: cleanstart/pgbouncer:latest
+    volumes:
+      - ./pgbouncer.ini:/etc/pgbouncer/pgbouncer.ini:ro
+      - ./userlist.txt:/etc/pgbouncer/userlist.txt:ro
+    command: ["/etc/pgbouncer/pgbouncer.ini"]
+    ports:
+      - ${PGBOUNCER_PORT:-6432}:6432
+    depends_on:
+      postgres:
+        condition: service_healthy
     networks:
       - intheloop-network
 

ecosystem.config.json

@@ -23,6 +23,12 @@
       "env": {
         "NODE_ENV": "development"
       }
+    },
+    {
+      "name": "pg-temp",
+      "script": "sh",
+      "args": "-c 'docker compose up postgres pgbouncer'",
+      "autorestart": false
     }
   ]
 }

eslint.config.js

@@ -360,6 +360,7 @@ export default [
       "scripts/**",
       "iframe-outputs/worker/dist/**",
       "iframe-outputs/worker/.wrangler/**",
+      "packages/livestore-postgres/dist/**",
     ],
   },
 ];

nginx.conf

@@ -17,7 +17,9 @@ server {
 
     # Proxy API requests to backend
     location /api {
-        proxy_pass http://${SYNC_HOST}:8787;
+        resolver 127.0.0.11 valid=30s;
+        set $sync_upstream http://${SYNC_HOST}:8787;
+        proxy_pass $sync_upstream;
         proxy_http_version 1.1;
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection 'upgrade';

package.json

@@ -67,6 +67,7 @@
     "@livestore/livestore": "^0.3.1",
     "@livestore/react": "^0.3.1",
     "@livestore/sync-cf": "^0.3.1",
+    "@runtimed/livestore-postgres": "workspace:*",
     "@livestore/wa-sqlite": "1.0.5-dev.2",
     "@livestore/webmesh": "^0.3.1",
     "@microlink/react-json-view": "^1.26.2",
@@ -119,6 +120,7 @@
     "nanoid": "^5.1.5",
     "openid-client": "^6.6.2",
     "path-to-regexp": "^8.2.0",
+    "pg": "^8.16.3",
     "psl": "^1.15.0",
     "react": "19.2.1",
     "react-dom": "19.2.1",
@@ -136,6 +138,7 @@
     "strip-ansi": "^7.1.0",
     "tailwind-merge": "^3.3.1",
     "ts-extras": "^0.16.0",
+    "tsx": "^4.21.0",
     "tw-animate-css": "^1.3.4",
     "ulid": "^3.0.1",
     "uuid": "^11.1.0",
@@ -158,6 +161,7 @@
     "@types/geojson": "^7946.0.16",
     "@types/js-cookie": "^3.0.6",
     "@types/node": "^24.0.10",
+    "@types/pg": "^8.16.0",
     "@types/react": "^19.1.8",
     "@types/react-dom": "^19.1.6",
     "@typescript-eslint/eslint-plugin": "^8.35.1",

packages/agent-core/src/artifact-client.ts

@@ -102,9 +102,9 @@ export class ArtifactClient implements IArtifactClient {
       });
 
       if (!response.ok) {
-        const error = await response.json().catch(() => ({
+        const error = (await response.json().catch(() => ({
           error: "Unknown error",
-        }));
+        }))) as { error?: string };
         throw new Error(
           `Artifact submission failed: ${error.error || response.statusText}`
         );

packages/agent-core/src/runtime-agent.ts

@@ -919,7 +919,7 @@ export class RuntimeAgent {
     const shutdown = () => this.shutdown();
 
     // Set up global error handlers (platform-agnostic)
-    globalThis.addEventListener("unhandledrejection", (event) => {
+    globalThis.addEventListener("unhandledrejection", (event: PromiseRejectionEvent) => {
       logger.error(
         "Unhandled rejection",
         event.reason instanceof Error ? event.reason : undefined,