[experiment/test] SQLite Bucket Storage (and MySQL bucket storage)

modules/module-drizzle-storage/README.md

@@ -0,0 +1,61 @@
+# Drizzle Bucket Storage
+
+Experimental Drizzle-backed bucket storage for the PowerSync service.
+
+The first implementation uses SQLite through `better-sqlite3`:
+
+```yaml
+storage:
+  type: drizzle:sqlite
+  filename: ./powersync-storage.sqlite
+```
+
+SQLite storage is restricted to the unified runner because checkpoint
+notifications are process-local. `:memory:` is supported for tests and
+throwaway development.
+
+## Design
+
+The SQLite driver owns its Drizzle table declarations. Column codecs map
+database values to the shared runtime shapes used by storage code, including
+lossless `bigint`, `Buffer`, JSON, boolean, and `Date` values. Compile-time
+assertions verify that the inferred SQLite query models match the canonical
+storage records.
+
+Storage classes use Drizzle queries directly. Database-specific behavior is
+kept behind the small dialect surface only where required: transactions,
+bucket reads, compaction/raw SQL, and checkpoint notification.
+
+## Migrations
+
+Drizzle Kit owns schema generation and migration metadata:
+
+```sh
+corepack pnpm --filter @powersync/service-module-drizzle-storage drizzle:generate:sqlite
+corepack pnpm --filter @powersync/service-module-drizzle-storage drizzle:check:sqlite
+```
+
+Generated SQL and metadata are committed under `src/migrations/sqlite` and
+copied into the built package. Runtime migrations are exposed through the
+normal PowerSync migration agent and guarded by a SQLite-backed lock.
+
+## Development
+
+```sh
+corepack pnpm --filter @powersync/service-module-drizzle-storage build
+corepack pnpm --filter @powersync/service-module-drizzle-storage build:tests
+corepack pnpm --filter @powersync/service-module-drizzle-storage test --run
+```
+
+PostgreSQL replication tests can opt into this storage with:
+
+```sh
+TEST_MONGO_STORAGE=false \
+TEST_POSTGRES_STORAGE=false \
+TEST_MIKROORM_SQLITE_STORAGE=false \
+TEST_DRIZZLE_SQLITE_STORAGE=true \
+corepack pnpm --filter @powersync/service-module-postgres test test/src/wal_stream.test.ts --run
+```
+
+Set `DRIZZLE_SQLITE_STORAGE_TEST_FILENAME` to override the file-backed test
+database path.

modules/module-drizzle-storage/drizzle.sqlite.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from 'drizzle-kit';
+
+export default defineConfig({
+  dialect: 'sqlite',
+  schema: './src/drivers/sqlite/schema.ts',
+  out: './src/migrations/sqlite'
+});

modules/module-drizzle-storage/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@powersync/service-module-drizzle-storage",
+  "repository": "https://github.com/powersync-ja/powersync-service",
+  "types": "dist/index.d.ts",
+  "version": "0.1.0",
+  "main": "dist/index.js",
+  "license": "FSL-1.1-ALv2",
+  "type": "module",
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "rm -rf ./dist ./tsconfig.tsbuildinfo && tsc -b && cp -R src/migrations/sqlite dist/migrations/sqlite",
+    "build:tests": "tsc -b test/tsconfig.json",
+    "clean": "rm -rf ./dist && tsc -b --clean",
+    "drizzle:generate:sqlite": "drizzle-kit generate --config=drizzle.sqlite.config.ts",
+    "drizzle:check:sqlite": "drizzle-kit check --config=drizzle.sqlite.config.ts",
+    "test": "vitest"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./types": {
+      "types": "./dist/types/types.d.ts",
+      "import": "./dist/types/types.js",
+      "require": "./dist/types/types.js",
+      "default": "./dist/types/types.js"
+    }
+  },
+  "dependencies": {
+    "@powersync/lib-services-framework": "workspace:*",
+    "@powersync/service-core": "workspace:*",
+    "@powersync/service-jsonbig": "workspace:*",
+    "@powersync/service-sync-rules": "workspace:*",
+    "@powersync/service-types": "workspace:*",
+    "better-sqlite3": "^12.10.0",
+    "drizzle-orm": "^0.45.2",
+    "ts-codec": "^1.3.0",
+    "uuid": "catalog:"
+  },
+  "devDependencies": {
+    "@powersync/service-core-tests": "workspace:*",
+    "@types/better-sqlite3": "^7.6.13",
+    "drizzle-kit": "^0.31.10",
+    "typescript": "catalog:"
+  }
+}

modules/module-drizzle-storage/src/drivers/sqlite/SqliteDrizzleStorageFactory.ts

@@ -0,0 +1,17 @@
+import { DrizzleBucketStorageFactory } from '../../storage/DrizzleBucketStorageFactory.js';
+import type { NormalizedDrizzleSqliteStorageConfig } from '../../types/types.js';
+import { createSqliteDrizzleRuntime, type SqliteDrizzleRuntime } from './sqlite-config.js';
+import { createSqliteDrizzleStorageDialect } from './sqlite-dialect.js';
+
+export function createSqliteDrizzleStorageFactory(options: {
+  config: NormalizedDrizzleSqliteStorageConfig;
+  slotNamePrefix: string;
+  runtime?: SqliteDrizzleRuntime;
+}): DrizzleBucketStorageFactory {
+  const runtime = options.runtime ?? createSqliteDrizzleRuntime(options.config);
+  return new DrizzleBucketStorageFactory({
+    runtime,
+    dialect: createSqliteDrizzleStorageDialect(runtime),
+    slotNamePrefix: options.slotNamePrefix
+  });
+}

modules/module-drizzle-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts

@@ -0,0 +1,76 @@
+import { locks } from '@powersync/lib-services-framework';
+import * as uuid from 'uuid';
+import type { SqliteDrizzleRuntime } from './sqlite-config.js';
+
+const DEFAULT_LOCK_TIMEOUT = 60_000;
+
+export class SqliteMigrationLockManager extends locks.AbstractLockManager {
+  constructor(private readonly options: locks.LockManagerParams & { runtime: SqliteDrizzleRuntime }) {
+    super(options);
+  }
+
+  private get timeout(): number {
+    return this.options.timeout ?? DEFAULT_LOCK_TIMEOUT;
+  }
+
+  async init(): Promise<void> {
+    this.options.runtime.client.exec(`
+      CREATE TABLE IF NOT EXISTS powersync_migration_locks (
+        name TEXT PRIMARY KEY,
+        lock_id TEXT,
+        expires_at INTEGER NOT NULL
+      )
+    `);
+    this.options.runtime.client
+      .prepare(
+        `
+      INSERT OR IGNORE INTO powersync_migration_locks (name, lock_id, expires_at)
+      VALUES (?, NULL, 0)
+    `
+      )
+      .run(this.options.name);
+  }
+
+  protected async acquireHandle(): Promise<locks.LockHandle | null> {
+    const lockId = uuid.v4();
+    const now = Date.now();
+    const result = this.options.runtime.client
+      .prepare(
+        `
+      UPDATE powersync_migration_locks
+      SET lock_id = ?, expires_at = ?
+      WHERE name = ? AND (lock_id IS NULL OR expires_at <= ?)
+    `
+      )
+      .run(lockId, now + this.timeout, this.options.name, now);
+    if (result.changes != 1) {
+      return null;
+    }
+    return {
+      refresh: async () => {
+        const refreshed = this.options.runtime.client
+          .prepare(
+            `
+          UPDATE powersync_migration_locks SET expires_at = ? WHERE name = ? AND lock_id = ?
+        `
+          )
+          .run(Date.now() + this.timeout, this.options.name, lockId);
+        if (refreshed.changes != 1) {
+          throw new Error('Lock not found, could not refresh');
+        }
+      },
+      release: async () => {
+        const released = this.options.runtime.client
+          .prepare(
+            `
+          UPDATE powersync_migration_locks SET lock_id = NULL, expires_at = 0 WHERE name = ? AND lock_id = ?
+        `
+          )
+          .run(this.options.name, lockId);
+        if (released.changes != 1) {
+          throw new Error('Lock not found, could not release');
+        }
+      }
+    };
+  }
+}

modules/module-drizzle-storage/src/drivers/sqlite/column-types.ts

@@ -0,0 +1,55 @@
+import { customType } from 'drizzle-orm/sqlite-core';
+
+/**
+ * SQLite INTEGER column whose application type is always bigint.
+ *
+ * The better-sqlite3 connections are configured with safe integers so the
+ * driver value reaches this mapper without first losing precision.
+ */
+export const sqliteBigInt = customType<{ data: bigint; driverData: bigint | number }>({
+  dataType() {
+    return 'bigint';
+  },
+  toDriver(value) {
+    return value;
+  },
+  fromDriver(value) {
+    return BigInt(value);
+  }
+});
+
+export const sqliteInteger = customType<{ data: number; driverData: bigint | number }>({
+  dataType() {
+    return 'integer';
+  },
+  toDriver(value) {
+    return value;
+  },
+  fromDriver(value) {
+    return Number(value);
+  }
+});
+
+export const sqliteBoolean = customType<{ data: boolean; driverData: bigint | number }>({
+  dataType() {
+    return 'integer';
+  },
+  toDriver(value) {
+    return value ? 1 : 0;
+  },
+  fromDriver(value) {
+    return value !== 0n && value !== 0;
+  }
+});
+
+export const sqliteTimestampMs = customType<{ data: Date; driverData: bigint | number }>({
+  dataType() {
+    return 'integer';
+  },
+  toDriver(value) {
+    return value.getTime();
+  },
+  fromDriver(value) {
+    return new Date(Number(value));
+  }
+});