Support SQLite as a scalar expression engine (#654)

Author: simolus3

.changeset/pretty-ravens-ring.md

@@ -0,0 +1,7 @@
+---
+'@powersync/service-core': minor
+'@powersync/service-sync-rules': minor
+'@powersync/service-module-mongodb-storage': patch
+---
+
+Add the experimental `unstable_sqlite_expression_engine` sync config option to evaluate Sync Streams with SQLite.

modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts

@@ -1,3 +1,5 @@
+import * as sqlite from 'node:sqlite';
+
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import {
@@ -7,6 +9,7 @@ import {
   DEFAULT_HYDRATION_STATE,
   HydratedSyncConfig,
   HydrationState,
+  nodeSqlite,
   ParameterIndexLookupCreator,
   ParameterLookupScope,
   SyncConfigWithErrors,
@@ -41,7 +44,10 @@ export class MongoPersistedSyncRules implements storage.PersistedSyncRules {
   }
 
   hydratedSyncConfig(): HydratedSyncConfig {
-    return this.syncConfigWithErrors.config.hydrate({ hydrationState: this.hydrationState });
+    return this.syncConfigWithErrors.config.hydrate({
+      hydrationState: this.hydrationState,
+      sqlite: nodeSqlite(sqlite)
+    });
   }
 }
 

modules/module-mongodb-storage/test/src/storage_sync.test.ts

@@ -1,7 +1,8 @@
 import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
-import { DEFAULT_HYDRATION_STATE, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
+import { DEFAULT_HYDRATION_STATE, nodeSqlite, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
+import * as sqlite from 'node:sqlite';
 import { describe, expect, test } from 'vitest';
 import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
 import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
@@ -55,7 +56,7 @@ function objectIdGenerator(id: string) {
 function hydratedRulesFor(yaml: string) {
   const parsed = SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS);
   expect(parsed.errors).toEqual([]);
-  return parsed.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE });
+  return parsed.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, sqlite: nodeSqlite(sqlite) });
 }
 
 function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {

packages/service-core-tests/src/tests/util.ts

@@ -4,7 +4,6 @@ import {
   ParameterLookupDefinitionId,
   ParameterLookupScope,
   SourceTableRef,
-  SqliteRow,
   TablePattern
 } from '@powersync/service-sync-rules';
 import { bucketRequest } from '../test-utils/general-utils.js';
@@ -36,8 +35,12 @@ const EMPTY_LOOKUP_SOURCE: ParameterIndexLookupCreator = {
   getSourceTables(): Set<TablePattern> {
     return new Set();
   },
-  evaluateParameterRow(_sourceTable: SourceTableRef, _row: SqliteRow) {
-    return [];
+  createEvaluator(input) {
+    return {
+      evaluateParameterRow(sourceTable, row) {
+        return [];
+      }
+    };
   },
   tableSyncsParameters(_table: SourceTableRef): boolean {
     return false;

packages/service-core/src/routes/endpoints/admin.ts

@@ -1,5 +1,7 @@
+import * as sqlite from 'node:sqlite';
+
 import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
-import { SourceSchema, SqlSyncRules, StaticSchema } from '@powersync/service-sync-rules';
+import { nodeSqlite, SourceSchema, SqlSyncRules, StaticSchema } from '@powersync/service-sync-rules';
 import { internal_routes } from '@powersync/service-types';
 
 import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
@@ -182,7 +184,10 @@ class FakeSyncRulesContentForValidation extends storage.PersistedSyncRulesConten
       }),
       hydrationState: DEFAULT_HYDRATION_STATE,
       hydratedSyncConfig() {
-        return this.syncConfigWithErrors.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE });
+        return this.syncConfigWithErrors.config.hydrate({
+          hydrationState: DEFAULT_HYDRATION_STATE,
+          sqlite: nodeSqlite(sqlite)
+        });
       }
     };
   }

packages/service-core/src/storage/PersistedSyncRulesContent.ts

@@ -7,14 +7,15 @@ import {
   ErrorLocation,
   HydratedSyncConfig,
   HydrationState,
-  javaScriptExpressionEngine,
+  nodeSqlite,
   PrecompiledSyncConfig,
   SqlEventDescriptor,
   SqlSyncRules,
   SyncConfigWithErrors,
   versionedHydrationState,
   YamlError
 } from '@powersync/service-sync-rules';
+import * as sqlite from 'node:sqlite';
 import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
 import { ReplicationLock } from './ReplicationLock.js';
 import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
@@ -101,7 +102,6 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
 
       const precompiled = new PrecompiledSyncConfig(plan, compatibility, eventDefinitions, {
         defaultSchema: options.defaultSchema,
-        engine: javaScriptExpressionEngine(compatibility),
         sourceText: this.sync_rules_content
       });
 
@@ -144,7 +144,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
       syncConfigWithErrors: config,
       hydrationState,
       hydratedSyncConfig: () => {
-        return config.config.hydrate({ hydrationState });
+        return config.config.hydrate({ hydrationState, sqlite: nodeSqlite(sqlite) });
       }
     };
   }

packages/service-core/test/src/routes/stream.test.ts

@@ -1,6 +1,12 @@
 import { BasicRouterRequest, Context, JwtPayload, SyncRulesBucketStorage } from '@/index.js';
-import { RouterResponse, ServiceError, logger } from '@powersync/lib-services-framework';
-import { DEFAULT_HYDRATION_STATE, SqlSyncRules } from '@powersync/service-sync-rules';
+import { logger, RouterResponse, ServiceError } from '@powersync/lib-services-framework';
+import {
+  DEFAULT_HYDRATION_STATE,
+  HydrateSyncConfigParams,
+  nodeSqlite,
+  SqlSyncRules
+} from '@powersync/service-sync-rules';
+import * as sqlite from 'node:sqlite';
 import { Readable, Writable } from 'stream';
 import { pipeline } from 'stream/promises';
 import { describe, expect, it } from 'vitest';
@@ -10,6 +16,11 @@ import { DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS, limitParamsForLogging } from '../
 import { mockServiceContext } from './mocks.js';
 
 describe('Stream Route', () => {
+  const defaultHydrationOptions: HydrateSyncConfigParams = {
+    hydrationState: DEFAULT_HYDRATION_STATE,
+    sqlite: nodeSqlite(sqlite)
+  };
+
   describe('compressed stream', () => {
     it('handles missing sync rules', async () => {
       const context: Context = {
@@ -45,7 +56,7 @@ describe('Stream Route', () => {
 
       const storage = {
         getParsedSyncRules() {
-          return new SqlSyncRules('bucket_definitions: {}').hydrate({ hydrationState: DEFAULT_HYDRATION_STATE });
+          return new SqlSyncRules('bucket_definitions: {}').hydrate(defaultHydrationOptions);
         },
         watchCheckpointChanges: async function* (options) {
           throw new Error('Simulated storage error');
@@ -83,7 +94,7 @@ describe('Stream Route', () => {
     it('logs the application metadata', async () => {
       const storage = {
         getParsedSyncRules() {
-          return new SqlSyncRules('bucket_definitions: {}').hydrate({ hydrationState: DEFAULT_HYDRATION_STATE });
+          return new SqlSyncRules('bucket_definitions: {}').hydrate(defaultHydrationOptions);
         },
         watchCheckpointChanges: async function* (options) {
           throw new Error('Simulated storage error');

packages/service-core/test/src/sync/BucketChecksumState.test.ts

@@ -15,17 +15,18 @@ import {
 } from '@/index.js';
 import { JSONBig } from '@powersync/service-jsonbig';
 import {
+  nodeSqlite,
   ParameterIndexLookupCreator,
   ParameterLookupDefinitionId,
   ParameterLookupRows,
   ParameterLookupScope,
   ScopedParameterLookup,
   SourceTableRef,
-  SqliteRow,
   SqlSyncRules,
   TablePattern,
   versionedHydrationState
 } from '@powersync/service-sync-rules';
+import * as sqlite from 'node:sqlite';
 import { beforeEach, describe, expect, test } from 'vitest';
 
 describe('BucketChecksumState', () => {
@@ -39,8 +40,12 @@ describe('BucketChecksumState', () => {
     getSourceTables(): Set<TablePattern> {
       return new Set();
     },
-    evaluateParameterRow(_sourceTable: SourceTableRef, _row: SqliteRow) {
-      return [];
+    createEvaluator() {
+      return {
+        evaluateParameterRow() {
+          return [];
+        }
+      };
     },
     tableSyncsParameters(_table: SourceTableRef): boolean {
       return false;
@@ -60,7 +65,7 @@ bucket_definitions:
     data: []
     `,
     { defaultSchema: 'public' }
-  ).config.hydrate({ hydrationState: versionedHydrationState(1) });
+  ).config.hydrate({ hydrationState: versionedHydrationState(1), sqlite: nodeSqlite(sqlite) });
 
   // global[1] and global[2]
   const SYNC_RULES_GLOBAL_TWO = SqlSyncRules.fromYaml(
@@ -73,7 +78,7 @@ bucket_definitions:
     data: []
     `,
     { defaultSchema: 'public' }
-  ).config.hydrate({ hydrationState: versionedHydrationState(2) });
+  ).config.hydrate({ hydrationState: versionedHydrationState(2), sqlite: nodeSqlite(sqlite) });
 
   // by_project[n]
   const SYNC_RULES_DYNAMIC = SqlSyncRules.fromYaml(
@@ -84,7 +89,7 @@ bucket_definitions:
     data: []
     `,
     { defaultSchema: 'public' }
-  ).config.hydrate({ hydrationState: versionedHydrationState(3) });
+  ).config.hydrate({ hydrationState: versionedHydrationState(3), sqlite: nodeSqlite(sqlite) });
 
   const syncContext = new SyncContext({
     maxBuckets: 100,
@@ -655,7 +660,7 @@ config:
 
       const rules = SqlSyncRules.fromYaml(source, {
         defaultSchema: 'public'
-      }).config.hydrate({ hydrationState: versionedHydrationState(1) });
+      }).config.hydrate({ hydrationState: versionedHydrationState(1), sqlite: nodeSqlite(sqlite) });
 
       return new BucketChecksumState({
         syncContext,
@@ -921,7 +926,8 @@ streams:
 `,
         { defaultSchema: 'public' }
       ).config.hydrate({
-        hydrationState: versionedHydrationState(1)
+        hydrationState: versionedHydrationState(1),
+        sqlite: nodeSqlite(sqlite)
       });
 
       const storage = new MockBucketChecksumStateStorage();
@@ -1018,7 +1024,8 @@ streams:
 `,
         { defaultSchema: 'public' }
       ).config.hydrate({
-        hydrationState: versionedHydrationState(1)
+        hydrationState: versionedHydrationState(1),
+        sqlite: nodeSqlite(sqlite)
       });
 
       const storage = new MockBucketChecksumStateStorage();
@@ -1090,7 +1097,7 @@ streams:
     query: SELECT id FROM comments WHERE p IN auth.parameter('c')
     `,
         { defaultSchema: 'public' }
-      ).config.hydrate({ hydrationState: versionedHydrationState(4) });
+      ).config.hydrate({ hydrationState: versionedHydrationState(4), sqlite: nodeSqlite(sqlite) });
 
       const storage = new MockBucketChecksumStateStorage();
 
@@ -1165,7 +1172,8 @@ streams:
       }
 
       const SYNC_RULES_MANY = SqlSyncRules.fromYaml(yamlDefinitions, { defaultSchema: 'public' }).config.hydrate({
-        hydrationState: versionedHydrationState(5)
+        hydrationState: versionedHydrationState(5),
+        sqlite: nodeSqlite(sqlite)
       });
 
       const storage = new MockBucketChecksumStateStorage();