diff --git a/docs/storage-v3.md b/docs/storage-v3.md new file mode 100644 index 000000000..8381d4be8 --- /dev/null +++ b/docs/storage-v3.md @@ -0,0 +1,78 @@ +# Storage version 3 - Data structure + +## Replication stream + +A replication stream represents one conceptual replication "job": + +1. One logical replication stream on the client in Postgres. +2. One change stream in MongoDB. +3. Generally, one entity creating checkpoints from a source database stream. + +This does not refer to concurrency - we may add concurrency in each of these streams at a later point, which may use multiple underlying database streams. Instead, it just refers to conceptually having one replication job, advancing checkpoints one at a time. + +Right now, each "sync config version", or `sync_rules` document, is one replication stream. + +For incremental reprocessing, this will change so that multiple sync config versions can be processed by the same stream. + +It is possible to have multiple replication streams running concurrently, for example when: + +1. Incremental reprocessing is not used, so nothing is shared between the sync config versions. +2. Changing storage versions - each replication stream can only handle one storage version at a time. + +## source_table + +Scoped to a replication stream. + +Collection: `source_table_${stream_id}` + +[FUTURE CHANGE] May have multiple copies per phyisical table per stream, especially when adding definitions to a stream. + +[FUTURE CHANGE] We can remove a source definition from a source table, but never add one. + +## source_records (previously current_data) + +Scoped to a source_table in a replication stream. + +Collection: `source_records_${stream_id}_${source_table_id}` + +The `_id` field is now the source row id. Unlike V1 storage model, this does not include `g` (group_id) or `t` (table id), since those are already encapsulated in the collection name. + +When a table is dropped, we first create relevant REMOVE operations, then drop the relevant current_data collection. + +[FUTURE CHANGE] If a _definition_ using a source table is removed: + +1. We remove the bucket_data (drop the collection - see below). +2. We _don't_ update the source-records collection - stale records will remain. (Purely because this would be a slow operation, without gaining much) +3. When re-processing a source record, we then check for orphaned references. + +When all definitions for a source table is removed, we remove the drop the corresponding source_records collection. + +## bucket_data + +Scoped by replication stream and definition id. + +Collection: `bucket_data_${stream_id}_${definition_id}` + +`_id.g` is removed, since this is encapsulated in the collection name now. + +`definition_id` is new here - that is not tracked in storage V1. + +[FUTURE CHANGE] collection must be dropped when the definition is removed. + +## parameter_index (previously bucket_parameters) + +Scoped by replication stream and index definition. + +Collection: `parameter_index_${stream_id}_${index_id}` + +_Also_ indexed by compound `key`, which includes {t: source_table_id, k: source_record_key} + +The `lookup` array drops the first two fields compared to V1 lookups (lookupName and queryId), since those are encapsulated in `index_id` in the collection name. In-memory, we use lookupName = indexId, queryId = '' (may change in the future). + +## bucket_state + +Scoped by replication stream. + +Collection: `bucket_state_${stream_id}`. + +`_id` is now compound: `{d: , b: }` (previously `{g, b}`) diff --git a/libs/lib-mongodb/src/db/mongo.ts b/libs/lib-mongodb/src/db/mongo.ts index b57d833f7..21678ff51 100644 --- a/libs/lib-mongodb/src/db/mongo.ts +++ b/libs/lib-mongodb/src/db/mongo.ts @@ -31,11 +31,13 @@ export const MONGO_OPERATION_TIMEOUT_MS = 40_000; export const MONGO_CHECKSUM_TIMEOUT_MS = 50_000; /** - * Same as above, but specifically for clear operations. + * Same as MONGO_OPERATION_TIMEOUT_MS, but specifically for clear operations. * * These are retried when reaching the timeout. + * + * Used to be 5s. Increased to attempt to improve efficiency (deleted documents / scanned documents). */ -export const MONGO_CLEAR_OPERATION_TIMEOUT_MS = 5_000; +export const MONGO_CLEAR_OPERATION_TIMEOUT_MS = MONGO_OPERATION_TIMEOUT_MS; export interface MongoConnectionOptions { maxPoolSize?: number; diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 10f0e61e2..07a426a4d 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -7,14 +7,16 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { generateSlotName } from '../utils/util.js'; +import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js'; +import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js'; +import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js'; import { PowerSyncMongo } from './implementation/db.js'; import { getMongoStorageConfig, SyncRuleDocument } from './implementation/models.js'; import { MongoChecksumOptions } from './implementation/MongoChecksums.js'; import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js'; -import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js'; export interface MongoBucketStorageOptions { - checksumOptions?: Omit; + checksumOptions?: Omit; } export class MongoBucketStorage extends storage.BucketStorageFactory { @@ -53,7 +55,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { id = Number(id); } const storageConfig = (syncRules as MongoPersistedSyncRulesContent).getStorageConfig(); - const storage = new MongoSyncBucketStorage( + const storage = createMongoSyncBucketStorage( this, id, syncRules as MongoPersistedSyncRulesContent, @@ -156,7 +158,6 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise { const storageVersion = options.storageVersion ?? storage.CURRENT_STORAGE_VERSION; const storageConfig = getMongoStorageConfig(storageVersion); - await this.db.initializeStorageVersion(storageConfig); let rules: MongoPersistedSyncRulesContent | undefined = undefined; @@ -205,6 +206,10 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { last_fatal_error_ts: null, last_keepalive_ts: null }; + if (storageConfig.incrementalReprocessing) { + const parsed = options.config.parsed; + doc.rule_mapping = BucketDefinitionMapping.fromParsedSyncRules(parsed).serialize(); + } await this.db.sync_rules.insertOne(doc); await this.db.notifyCheckpoint(); rules = new MongoPersistedSyncRulesContent(this.db, doc); @@ -296,63 +301,90 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { } }; - const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' }); - if (active_sync_rules == null) { - return { - operations_size_bytes: 0, - parameters_size_bytes: 0, - replication_size_bytes: 0 - }; - } - const operations_aggregate = await this.db.bucket_data + // For now, we get storage metrics over all v1 and v3 collections. + // In the future, we may split these metrics to report separately for active replication streams versus processing streams. - .aggregate([ - { - $collStats: { - storageStats: {} - } - } - ]) - .toArray() - .catch(ignoreNotExisting); + const aggregateStaticCollection = async (collection: mongo.Collection) => { + // We check whether the collection exists before getting the statistics. This avoids repeated + // errors in the MongoDB logs if the collection hasn't been created yet. + const exists = + (await this.db.db.listCollections({ name: collection.collectionName }, { nameOnly: true }).toArray()).length > + 0; + if (!exists) { + return [{ storageStats: { size: 0 } }]; + } - const parameters_aggregate = await this.db.bucket_parameters - .aggregate([ - { - $collStats: { - storageStats: {} + return collection + .aggregate([ + { + $collStats: { + storageStats: {} + } } - } - ]) - .toArray() - .catch(ignoreNotExisting); + ]) + .toArray() + .catch(ignoreNotExisting); + }; - const v1_replication_aggregate = await this.db.current_data - .aggregate([ - { - $collStats: { - storageStats: {} - } - } - ]) - .toArray() - .catch(ignoreNotExisting); + const operations_aggregate = await aggregateStaticCollection(this.db.bucket_data); + const v3_operation_aggregates = await Promise.all( + (await this.db.listBucketDataCollectionsV3()).map((collection) => + collection + .aggregate([ + { + $collStats: { + storageStats: {} + } + } + ]) + .toArray() + .catch(ignoreNotExisting) + ) + ); - const v3_replication_aggregate = await this.db.v3_current_data - .aggregate([ - { - $collStats: { - storageStats: {} - } - } - ]) - .toArray() - .catch(ignoreNotExisting); + const parameters_aggregate = await aggregateStaticCollection(this.db.bucket_parameters); + + const v3_parameter_aggregates = await Promise.all( + (await this.db.listAllParameterIndexCollectionsV3()).map((collection) => + collection + .aggregate([ + { + $collStats: { + storageStats: {} + } + } + ]) + .toArray() + .catch(ignoreNotExisting) + ) + ); + + const v1_source_record_aggregate = await aggregateStaticCollection(this.db.current_data); + + const source_record_aggregates = await Promise.all( + (await this.db.listAllSourceRecordCollectionsV3()).map((collection) => + collection + .aggregate([ + { + $collStats: { + storageStats: {} + } + } + ]) + .toArray() + .catch(ignoreNotExisting) + ) + ); return { - operations_size_bytes: Number(operations_aggregate[0].storageStats.size), - parameters_size_bytes: Number(parameters_aggregate[0].storageStats.size), + operations_size_bytes: + Number(operations_aggregate[0].storageStats.size) + + v3_operation_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0), + parameters_size_bytes: + Number(parameters_aggregate[0].storageStats.size) + + v3_parameter_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0), replication_size_bytes: - Number(v1_replication_aggregate[0].storageStats.size) + Number(v3_replication_aggregate[0].storageStats.size) + Number(v1_source_record_aggregate[0]?.storageStats?.size ?? 0) + + source_record_aggregates.reduce((total, aggregate) => total + Number(aggregate[0]?.storageStats?.size ?? 0), 0) }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts new file mode 100644 index 000000000..fc7bfd672 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts @@ -0,0 +1,72 @@ +import { ServiceAssertionError } from '@powersync/lib-services-framework'; +import { BucketDataSource, ParameterIndexLookupCreator, SyncConfigWithErrors } from '@powersync/service-sync-rules'; +import { SyncRuleDocument } from './models.js'; + +export type BucketDefinitionId = string; +export type ParameterIndexId = string; + +export class BucketDefinitionMapping { + static fromSyncRules(doc: Pick): BucketDefinitionMapping { + return new BucketDefinitionMapping(doc.rule_mapping?.definitions ?? {}, doc.rule_mapping?.parameter_indexes ?? {}); + } + + static fromParsedSyncRules(syncRules: SyncConfigWithErrors): BucketDefinitionMapping { + const definitionNames = syncRules.config.bucketDataSources.map((source) => source.uniqueName).sort(); + const parameterKeys = syncRules.config.bucketParameterLookupSources + .map((source) => `${source.defaultLookupScope.lookupName}#${source.defaultLookupScope.queryId}`) + .sort(); + + const definitions: Record = {}; + const parameterLookups: Record = {}; + + for (const [index, uniqueName] of definitionNames.entries()) { + definitions[uniqueName] = (index + 1).toString(16); + } + for (const [index, key] of parameterKeys.entries()) { + parameterLookups[key] = (index + 1).toString(16); + } + + return new BucketDefinitionMapping(definitions, parameterLookups); + } + + constructor( + private definitions: Record = {}, + private parameterLookupMapping: Record = {} + ) {} + + bucketSourceId(source: BucketDataSource): BucketDefinitionId { + const defId = this.definitions[source.uniqueName]; + if (defId == null) { + throw new ServiceAssertionError(`No mapping found for bucket source ${source.uniqueName}`); + } + return defId; + } + + allBucketDefinitionIds(): BucketDefinitionId[] { + return Object.values(this.definitions); + } + + allParameterIndexIds(): ParameterIndexId[] { + return Object.values(this.parameterLookupMapping); + } + + parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId { + const key = this.parameterLookupKey(source.defaultLookupScope.lookupName, source.defaultLookupScope.queryId); + const defId = this.parameterLookupMapping[key]; + if (defId == null) { + throw new ServiceAssertionError(`No mapping found for parameter lookup source ${key}`); + } + return defId; + } + + private parameterLookupKey(lookupName: string, queryId: string) { + return `${lookupName}#${queryId}`; + } + + serialize(): NonNullable { + return { + definitions: { ...this.definitions }, + parameter_indexes: { ...this.parameterLookupMapping } + }; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 5b25d18f2..770d94a11 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -24,18 +24,16 @@ import { utils } from '@powersync/service-core'; import * as timers from 'node:timers/promises'; -import { idPrefixFilter, mongoTableId } from '../../utils/util.js'; -import { VersionedPowerSyncMongo } from './db.js'; -import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js'; +import { mongoTableId } from '../../utils/util.js'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { PersistedBatch } from './common/PersistedBatch.js'; +import { LoadedSourceRecord, SourceRecordStore } from './common/SourceRecordStore.js'; +import type { VersionedPowerSyncMongo } from './db.js'; +import { SyncRuleDocument } from './models.js'; +import { MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; -import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; -import { PersistedBatch } from './PersistedBatch.js'; - -/** - * 15MB - */ -export const MAX_ROW_SIZE = 15 * 1024 * 1024; +import { OperationBatch, RecordOperation } from './OperationBatch.js'; // Currently, we can only have a single flush() at a time, since it locks the op_id sequence. // While the MongoDB transaction retry mechanism handles this okay, using an in-process Mutex @@ -44,8 +42,6 @@ export const MAX_ROW_SIZE = 15 * 1024 * 1024; // In the future, we can investigate allowing multiple replication streams operating independently. const replicationMutex = new utils.Mutex(); -export const EMPTY_DATA = new bson.Binary(bson.serialize({})); - export interface MongoBucketBatchOptions { db: VersionedPowerSyncMongo; syncRules: HydratedSyncRules; @@ -55,6 +51,7 @@ export interface MongoBucketBatchOptions { keepaliveOp: InternalOpId | null; resumeFromLsn: string | null; storeCurrentData: boolean; + mapping: BucketDefinitionMapping; /** * Set to true for initial replication. */ @@ -65,22 +62,23 @@ export interface MongoBucketBatchOptions { logger?: Logger; } -export class MongoBucketBatch +export abstract class MongoBucketBatch extends BaseObserver implements storage.BucketStorageBatch { - private logger: Logger; + protected logger: Logger; private readonly client: mongo.MongoClient; public readonly db: VersionedPowerSyncMongo; public readonly session: mongo.ClientSession; private readonly sync_rules: HydratedSyncRules; - private readonly group_id: number; + protected readonly group_id: number; private readonly slot_name: string; private readonly storeCurrentData: boolean; private readonly skipExistingRows: boolean; + protected readonly mapping: BucketDefinitionMapping; private batch: OperationBatch | null = null; private write_checkpoint_batch: storage.CustomWriteCheckpointOptions[] = []; @@ -129,6 +127,7 @@ export class MongoBucketBatch this.slot_name = options.slotName; this.sync_rules = options.syncRules; this.storeCurrentData = options.storeCurrentData; + this.mapping = options.mapping; this.skipExistingRows = options.skipExistingRows; this.markRecordUnavailable = options.markRecordUnavailable; this.batch = new OperationBatch(); @@ -147,6 +146,12 @@ export class MongoBucketBatch return this.last_checkpoint_lsn; } + protected abstract createPersistedBatch(writtenSize: number): PersistedBatch; + + protected abstract get sourceRecordStore(): SourceRecordStore; + + protected abstract cleanupDroppedSourceTables(sourceTables: storage.SourceTable[]): Promise; + async flush(options?: storage.BatchBucketFlushOptions): Promise { let result: storage.FlushedResult | null = null; // One flush may be split over multiple transactions. @@ -212,33 +217,12 @@ export class MongoBucketBatch // (automatically limited to 48MB(?) per batch by MongoDB). The issue is that it changes // the order of processing, which then becomes really tricky to manage. // This now takes 2+ queries, but doesn't have any issues with order of operations. - const sizeLookups: SourceKey[] = batch.batch.map((r) => { - return { g: this.group_id, t: mongoTableId(r.record.sourceTable.id), k: r.beforeId }; - }); - - sizes = new Map(); + const sizeLookups = batch.batch.map((r) => ({ + sourceTableId: mongoTableId(r.record.sourceTable.id), + replicaId: r.beforeId + })); - const sizeCursor: mongo.AggregationCursor<{ _id: SourceKey; size: number }> = - this.db.common_current_data.aggregate( - [ - { - $match: { - _id: { $in: sizeLookups } - } - }, - { - $project: { - _id: 1, - size: { $bsonSize: '$$ROOT' } - } - } - ], - { session } - ); - for await (let doc of sizeCursor.stream()) { - const key = cacheKey(doc._id.t, doc._id.k); - sizes.set(key, doc.size); - } + sizes = await this.sourceRecordStore.loadSizes(session, sizeLookups); } // If set, we need to start a new transaction with this batch. @@ -256,41 +240,29 @@ export class MongoBucketBatch } continue; } - const lookups: SourceKey[] = b.map((r) => { - return { g: this.group_id, t: mongoTableId(r.record.sourceTable.id), k: r.beforeId }; - }); - let current_data_lookup = new Map(); - // With skipExistingRows, we only need to know whether or not the row exists. - const projection = this.skipExistingRows ? { _id: 1 } : undefined; - const cursor = this.db.common_current_data.find( - { - _id: { $in: lookups } - }, - { session, projection } - ); - for await (let doc of cursor.stream()) { - current_data_lookup.set(cacheKey(doc._id.t, doc._id.k), doc); - } + const lookups = b.map((r) => ({ + sourceTableId: mongoTableId(r.record.sourceTable.id), + replicaId: r.beforeId + })); + let sourceRecordLookup = await this.sourceRecordStore.loadDocuments(session, lookups, this.skipExistingRows); - let persistedBatch: PersistedBatch | null = new PersistedBatch(this.db, this.group_id, transactionSize, { - logger: this.logger - }); + let persistedBatch: PersistedBatch | null = this.createPersistedBatch(transactionSize); for (let op of b) { if (resumeBatch) { resumeBatch.push(op); continue; } - const currentData = current_data_lookup.get(op.internalBeforeKey) ?? null; - if (currentData != null) { + const sourceRecord = sourceRecordLookup.get(op.internalBeforeKey) ?? null; + if (sourceRecord != null) { // If it will be used again later, it will be set again using nextData below - current_data_lookup.delete(op.internalBeforeKey); + sourceRecordLookup.delete(op.internalBeforeKey); } - const nextData = this.saveOperation(persistedBatch!, op, currentData, op_seq); + const nextData = this.saveOperation(persistedBatch!, op, sourceRecord, op_seq); if (nextData != null) { // Update our current_data and size cache - current_data_lookup.set(op.internalAfterKey!, nextData); - sizes?.set(op.internalAfterKey!, nextData.data.length()); + sourceRecordLookup.set(op.internalAfterKey!, nextData); + sizes?.set(op.internalAfterKey!, nextData.data?.length() ?? 0); } if (persistedBatch!.shouldFlushTransaction()) { @@ -323,7 +295,7 @@ export class MongoBucketBatch private saveOperation( batch: PersistedBatch, operation: RecordOperation, - current_data: CurrentDataDocument | null, + sourceRecord: LoadedSourceRecord | null, opSeq: MongoIdSequence ) { const record = operation.record; @@ -332,16 +304,16 @@ export class MongoBucketBatch let after = record.after; const sourceTable = record.sourceTable; - let existing_buckets: CurrentBucket[] = []; - let new_buckets: CurrentBucket[] = []; - let existing_lookups: bson.Binary[] = []; - let new_lookups: bson.Binary[] = []; + let existing_buckets: LoadedSourceRecord['buckets'] = []; + let new_buckets: LoadedSourceRecord['buckets'] = []; + let existing_lookups: LoadedSourceRecord['lookups'] = []; + let new_lookups: LoadedSourceRecord['lookups'] = []; - const before_key: SourceKey = { g: this.group_id, t: mongoTableId(record.sourceTable.id), k: beforeId }; + const sourceTableId = mongoTableId(record.sourceTable.id); if (this.skipExistingRows) { if (record.tag == SaveOperationTag.INSERT) { - if (current_data != null) { + if (sourceRecord != null) { // Initial replication, and we already have the record. // This may be a different version of the record, but streaming replication // will take care of that. @@ -354,7 +326,7 @@ export class MongoBucketBatch } if (record.tag == SaveOperationTag.UPDATE) { - const result = current_data; + const result = sourceRecord; if (result == null) { // Not an error if we re-apply a transaction existing_buckets = []; @@ -375,13 +347,13 @@ export class MongoBucketBatch } else { existing_buckets = result.buckets; existing_lookups = result.lookups; - if (this.storeCurrentData) { - const data = deserializeBson((result.data as mongo.Binary).buffer) as SqliteRow; + if (this.storeCurrentData && result.data != null) { + const data = deserializeBson(result.data.buffer) as SqliteRow; after = storage.mergeToast(after!, data); } } } else if (record.tag == SaveOperationTag.DELETE) { - const result = current_data; + const result = sourceRecord; if (result == null) { // Not an error if we re-apply a transaction existing_buckets = []; @@ -398,9 +370,9 @@ export class MongoBucketBatch } } - let afterData: bson.Binary | undefined; + let afterData: bson.Binary | null = null; if (afterId != null && !this.storeCurrentData) { - afterData = EMPTY_DATA; + afterData = null; } else if (afterId != null) { try { // This will fail immediately if the record is > 16MB. @@ -498,13 +470,7 @@ export class MongoBucketBatch table: sourceTable, before_buckets: existing_buckets }); - new_buckets = evaluated.map((e) => { - return { - bucket: e.bucket, - table: e.table, - id: e.id - }; - }); + new_buckets = this.sourceRecordStore.mapEvaluatedBuckets(evaluated); } if (sourceTable.syncParameters) { @@ -537,28 +503,29 @@ export class MongoBucketBatch evaluated: paramEvaluated, existing_lookups }); - new_lookups = paramEvaluated.map((p) => { - return storage.serializeLookup(p.lookup); - }); + new_lookups = this.sourceRecordStore.mapParameterLookups(paramEvaluated); } } - let result: CurrentDataDocument | null = null; + let result: LoadedSourceRecord | null = null; // 5. TOAST: Update current data and bucket list. if (afterId) { // Insert or update - const after_key: SourceKey = { g: this.group_id, t: mongoTableId(sourceTable.id), k: afterId }; - batch.upsertCurrentData(after_key, { + batch.upsertCurrentData({ + sourceTableId, + replicaId: afterId, data: afterData, buckets: new_buckets, lookups: new_lookups }); result = { - _id: after_key, - data: afterData!, + sourceTableId, + replicaId: afterId, + data: afterData, buckets: new_buckets, - lookups: new_lookups + lookups: new_lookups, + cacheKey: operation.internalAfterKey! }; } @@ -567,7 +534,7 @@ export class MongoBucketBatch // Note that this is a soft delete. // We don't specifically need a new or unique op_id here, but it must be greater than the // last checkpoint, so we use next(). - batch.softDeleteCurrentData(before_key, opSeq.next()); + batch.softDeleteCurrentData(sourceTableId, beforeId, opSeq.next()); } return result; } @@ -848,25 +815,13 @@ export class MongoBucketBatch await this.db.notifyCheckpoint(); this.persisted_op = null; this.last_checkpoint_lsn = lsn; - if (this.db.storageConfig.softDeleteCurrentData && newLastCheckpoint != null) { - await this.cleanupCurrentData(newLastCheckpoint); + if (newLastCheckpoint != null) { + await this.sourceRecordStore.postCommitCleanup(newLastCheckpoint, this.logger); } } return { checkpointBlocked, checkpointCreated }; } - private async cleanupCurrentData(lastCheckpoint: bigint) { - const result = await this.db.v3_current_data.deleteMany({ - '_id.g': this.group_id, - pending_delete: { $exists: true, $lte: lastCheckpoint } - }); - if (result.deletedCount > 0) { - this.logger.info( - `Cleaned up ${result.deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}` - ); - } - } - /** * Switch from processing -> active if relevant. * @@ -988,9 +943,11 @@ export class MongoBucketBatch await this.withTransaction(async () => { for (let table of sourceTables) { - await this.db.source_tables.deleteOne({ _id: mongoTableId(table.id) }); + await this.db.commonSourceTables(this.group_id).deleteOne({ _id: mongoTableId(table.id) }); } }); + + await this.cleanupDroppedSourceTables(sourceTables); return result; } @@ -1022,24 +979,9 @@ export class MongoBucketBatch let lastBatchCount = BATCH_LIMIT; while (lastBatchCount == BATCH_LIMIT) { await this.withReplicationTransaction(`Truncate ${sourceTable.qualifiedName}`, async (session, opSeq) => { - const current_data_filter: mongo.Filter = { - _id: idPrefixFilter({ g: this.group_id, t: mongoTableId(sourceTable.id) }, ['k']), - // Skip soft-deleted data - // Works for both v1 and v3 current_data schemas - pending_delete: { $exists: false } - }; - - const cursor = this.db.common_current_data.find(current_data_filter, { - projection: { - _id: 1, - buckets: 1, - lookups: 1 - }, - limit: BATCH_LIMIT, - session: session - }); - const batch = await cursor.toArray(); - const persistedBatch = new PersistedBatch(this.db, this.group_id, 0, { logger: this.logger }); + const sourceTableId = mongoTableId(sourceTable.id); + const batch = await this.sourceRecordStore.loadTruncateBatch(session, sourceTableId, BATCH_LIMIT); + const persistedBatch = this.createPersistedBatch(0); for (let value of batch) { persistedBatch.saveBucketData({ @@ -1047,18 +989,18 @@ export class MongoBucketBatch before_buckets: value.buckets, evaluated: [], table: sourceTable, - sourceKey: value._id.k + sourceKey: value.replicaId }); persistedBatch.saveParameterData({ op_seq: opSeq, existing_lookups: value.lookups, evaluated: [], sourceTable: sourceTable, - sourceKey: value._id.k + sourceKey: value.replicaId }); // Since this is not from streaming replication, we can do a hard delete - persistedBatch.hardDeleteCurrentData(value._id); + persistedBatch.hardDeleteCurrentData(sourceTableId, value.replicaId); } await persistedBatch.flush(session); lastBatchCount = batch.length; @@ -1083,7 +1025,7 @@ export class MongoBucketBatch copy.snapshotStatus = snapshotStatus; await this.withTransaction(async () => { - await this.db.source_tables.updateOne( + await this.db.commonSourceTables(this.group_id).updateOne( { _id: mongoTableId(table.id) }, { $set: { @@ -1138,7 +1080,7 @@ export class MongoBucketBatch const ids = tables.map((table) => mongoTableId(table.id)); await this.withTransaction(async () => { - await this.db.source_tables.updateMany( + await this.db.commonSourceTables(this.group_id).updateMany( { _id: { $in: ids } }, { $set: { @@ -1204,7 +1146,3 @@ export class MongoBucketBatch ); } } - -export function currentBucketKey(b: CurrentBucket) { - return `${b.bucket}/${b.table}/${b.id}`; -} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts new file mode 100644 index 000000000..54ee38691 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts @@ -0,0 +1,11 @@ +import * as bson from 'bson'; +import { SourceRecordBucketState } from './common/SourceRecordStore.js'; + +export const MAX_ROW_SIZE = 15 * 1024 * 1024; + +export const EMPTY_DATA = new bson.Binary(bson.serialize({})); + +export function currentBucketKey(bucket: SourceRecordBucketState) { + const prefix = bucket.definitionId == null ? '' : `${bucket.definitionId}:`; + return `${prefix}${bucket.bucket}/${bucket.table}/${bucket.id}`; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts index 8add91601..f12d2c722 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts @@ -1,4 +1,3 @@ -import * as lib_mongo from '@powersync/lib-service-mongodb'; import { addPartialChecksums, bson, @@ -13,8 +12,24 @@ import { PartialChecksumMap, PartialOrFullChecksum } from '@powersync/service-core'; -import { VersionedPowerSyncMongo } from './db.js'; -import { StorageConfig } from './models.js'; +import type { VersionedPowerSyncMongo } from './db.js'; + +import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { BucketDefinitionId, BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { BucketDataDocumentBase, StorageConfig } from './models.js'; + +export interface FetchPartialBucketChecksumV3 { + bucket: string; + definitionId: BucketDefinitionId; + start?: InternalOpId; + end: InternalOpId; +} + +export interface FetchPartialBucketChecksumByBucket { + bucket: string; + start?: InternalOpId; + end: InternalOpId; +} /** * Checksum calculation options, primarily for tests. @@ -31,28 +46,20 @@ export interface MongoChecksumOptions { operationBatchLimit?: number; storageConfig: StorageConfig; + mapping?: BucketDefinitionMapping; } const DEFAULT_BUCKET_BATCH_LIMIT = 200; const DEFAULT_OPERATION_BATCH_LIMIT = 50_000; -/** - * Checksum query implementation. - * - * General implementation flow is: - * 1. getChecksums() -> check cache for (partial) matches. If not found or partial match, query the remainder using computePartialChecksums(). - * 2. computePartialChecksums() -> query bucket_state for partial matches. Query the remainder using computePartialChecksumsDirect(). - * 3. computePartialChecksumsDirect() -> split into batches of 200 buckets at a time -> computePartialChecksumsInternal() - * 4. computePartialChecksumsInternal() -> aggregate over 50_000 operations in bucket_data at a time - */ -export class MongoChecksums { +export abstract class MongoChecksums { private _cache: ChecksumCache | undefined; private readonly storageConfig: StorageConfig; constructor( - private db: VersionedPowerSyncMongo, - private group_id: number, - private options: MongoChecksumOptions + protected readonly db: VersionedPowerSyncMongo, + protected readonly group_id: number, + protected readonly options: MongoChecksumOptions ) { this.storageConfig = options.storageConfig; } @@ -96,41 +103,7 @@ export class MongoChecksums { if (batch.length == 0) { return new Map(); } - - const preFilters: any[] = []; - for (let request of batch) { - if (request.start == null) { - preFilters.push({ - _id: { - g: this.group_id, - b: request.bucket - }, - 'compacted_state.op_id': { $exists: true, $lte: request.end } - }); - } - } - - const preStates = new Map(); - - if (preFilters.length > 0) { - // For un-cached bucket checksums, attempt to use the compacted state first. - const states = await this.db.bucket_state - .find({ - $or: preFilters - }) - .toArray(); - for (let state of states) { - const compactedState = state.compacted_state!; - preStates.set(state._id.b, { - opId: compactedState.op_id, - checksum: { - bucket: state._id.b, - checksum: Number(compactedState.checksum), - count: compactedState.count - } - }); - } - } + const preStates = await this.fetchPreStates(batch); const mappedRequests = batch.map((request) => { let start = request.start; @@ -199,11 +172,24 @@ export class MongoChecksums { * * `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this. */ - private async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { + protected abstract computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise; + + protected abstract fetchPreStates( + batch: FetchPartialBucketChecksum[] + ): Promise>; + + protected async computePartialChecksumsForCollection< + TRequest extends FetchPartialBucketChecksumByBucket, + TBucketDataDocument extends BucketDataDocumentBase + >( + batch: TRequest[], + collection: lib_mongo.mongo.Collection, + createFilter: (request: TRequest) => any + ): Promise { const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT; // Map requests by bucket. We adjust this as we get partial results. - let requests = new Map(); + let requests = new Map(); for (let request of batch) { requests.set(request.bucket, request); } @@ -211,23 +197,7 @@ export class MongoChecksums { const partialChecksums = new Map(); while (requests.size > 0) { - const filters: any[] = []; - for (let request of requests.values()) { - filters.push({ - _id: { - $gt: { - g: this.group_id, - b: request.bucket, - o: request.start ?? new bson.MinKey() - }, - $lte: { - g: this.group_id, - b: request.bucket, - o: request.end - } - } - }); - } + const filters = Array.from(requests.values(), createFilter); // Historically, checksum may be stored as 'int' or 'double'. // More recently, this should be a 'long'. @@ -243,7 +213,7 @@ export class MongoChecksums { // Returns: B[3-10], C[1-4] // 3. Query: C[5-end] // Returns: C[5-10] - const aggregate = await this.db.bucket_data + const aggregate = await collection .aggregate( [ { @@ -298,10 +268,8 @@ export class MongoChecksums { limitReached = true; const req = requests.get(bucket); requests.set(bucket, { - bucket, - source: req!.source, - start: doc.last_op, - end: req!.end + ...req!, + start: doc.last_op }); } else { // All done for this bucket @@ -339,6 +307,15 @@ export class MongoChecksums { } } +export function emptyChecksumForRequest( + request: Pick +): PartialOrFullChecksum { + if (request.start == null) { + return { bucket: request.bucket, count: 0, checksum: 0 }; + } + return { bucket: request.bucket, partialCount: 0, partialChecksum: 0 }; +} + /** * Convert output of the $group stage into a checksum. */ diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index 76000aa75..78f6fb384 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -9,15 +9,18 @@ import { utils } from '@powersync/service-core'; -import { VersionedPowerSyncMongo } from './db.js'; -import { BucketDataDocument, BucketDataKey, BucketStateDocument } from './models.js'; -import { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js'; +import { BucketDefinitionId } from './BucketDefinitionMapping.js'; +import { BucketDataDoc, BucketKey } from './common/BucketDataDoc.js'; +import { BucketDataDocumentGeneric, SingleBucketStore } from './common/SingleBucketStore.js'; +import type { VersionedPowerSyncMongo } from './db.js'; +import { BucketStateDocumentBase } from './models.js'; +import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js'; import { cacheKey } from './OperationBatch.js'; interface CurrentBucketState { /** Bucket name */ bucket: string; - + definitionId: BucketDefinitionId; /** * Rows seen in the bucket, with the last op_id of each. */ @@ -26,36 +29,30 @@ interface CurrentBucketState { * Estimated memory usage of the seen Map. */ trackingSize: number; - /** * Last (lowest) seen op_id that is not a PUT. */ lastNotPut: InternalOpId | null; - /** * Number of REMOVE/MOVE operations seen since lastNotPut. */ opsSincePut: number; - /** - * Incrementally-updated checksum, up to maxOpId + * Incrementally-updated checksum, up to maxOpId. */ checksum: number; - /** - * op count for the checksum + * Op count for the checksum. */ opCount: number; - /** * Byte size of ops covered by the checksum. */ opBytes: number; } -/** - * Additional options, primarily for testing. - */ +type CompactClearProperties = 'op' | 'checksum' | 'target_op'; + export interface MongoCompactOptions extends storage.CompactOptions {} const DEFAULT_CLEAR_BATCH_LIMIT = 5000; @@ -64,28 +61,34 @@ const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; const DEFAULT_MIN_BUCKET_CHANGES = 10; const DEFAULT_MIN_CHANGE_RATIO = 0.1; const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000; - /** This default is primarily for tests. */ const DEFAULT_MEMORY_LIMIT_MB = 64; -export class MongoCompactor { - private updates: mongo.AnyBulkWriteOperation[] = []; - private bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; - - private idLimitBytes: number; - private moveBatchLimit: number; - private moveBatchQueryLimit: number; - private clearBatchLimit: number; - private minBucketChanges: number; - private minChangeRatio: number; - private maxOpId: bigint; - private buckets: string[] | undefined; - private signal?: AbortSignal; - private group_id: number; +export interface DirtyBucket { + bucket: string; + definitionId: BucketDefinitionId | null; + estimatedCount: number; + dirtyRatio?: number; +} + +export abstract class MongoCompactor { + protected updates: mongo.AnyBulkWriteOperation[] = []; + protected bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; + + protected readonly idLimitBytes: number; + protected readonly moveBatchLimit: number; + protected readonly moveBatchQueryLimit: number; + protected readonly clearBatchLimit: number; + protected readonly minBucketChanges: number; + protected readonly minChangeRatio: number; + protected readonly maxOpId: bigint; + protected readonly buckets: string[] | undefined; + protected readonly signal?: AbortSignal; + protected readonly group_id: number; constructor( - private storage: MongoSyncBucketStorage, - private db: VersionedPowerSyncMongo, + protected readonly storage: MongoSyncBucketStorage, + protected readonly db: VersionedPowerSyncMongo, options: MongoCompactOptions ) { this.group_id = storage.group_id; @@ -107,9 +110,8 @@ export class MongoCompactor { */ async compact() { if (this.buckets) { - for (let bucket of this.buckets) { - // We can make this more efficient later on by iterating - // through the buckets in a single query. + for (const bucket of this.buckets) { + // We can make this more efficient later on by iterating through the buckets in a single query. // That makes batching more tricky, so we leave for later. await this.compactSingleBucketRetried(bucket); } @@ -118,8 +120,161 @@ export class MongoCompactor { } } - private async compactDirtyBuckets() { - for await (let buckets of this.dirtyBucketBatches({ + /** + * Subset of compact, only populating checksums where relevant. + */ + async populateChecksums(options: { minBucketChanges: number }): Promise { + let count = 0; + while (true) { + this.signal?.throwIfAborted(); + const buckets = await this.dirtyBucketBatchForChecksums(options); + if (buckets.length == 0) { + break; + } + this.signal?.throwIfAborted(); + + const start = Date.now(); + // Filter batch by estimated bucket size, to reduce possibility of timeouts. + const checkBuckets: typeof buckets = []; + let totalCountEstimate = 0; + for (const bucket of buckets) { + checkBuckets.push(bucket); + totalCountEstimate += bucket.estimatedCount; + if (totalCountEstimate > 50_000) { + break; + } + } + logger.info( + `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` + ); + await this.updateChecksumsBatch(checkBuckets); + logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); + count += checkBuckets.length; + } + return { buckets: count }; + } + + protected async *dirtyBucketBatchesForCollection( + collection: mongo.Collection, + lastId: TCollectionBucketState['_id'], + maxId: TCollectionBucketState['_id'], + options: { + minBucketChanges: number; + minChangeRatio: number; + }, + getDefinitionId: (state: TCollectionBucketState) => BucketDefinitionId | null + ): AsyncGenerator { + while (true) { + // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline + // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria. + const [result] = await collection + .aggregate<{ + buckets: TCollectionBucketState[]; + cursor: Pick[]; + }>( + [ + { + $match: { + _id: { $gt: lastId, $lt: maxId } + } + }, + { + $sort: { _id: 1 } + }, + { + // Scan a fixed number of docs each query so sparse matches don't block progress. + $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE + }, + { + $facet: { + buckets: [ + { + $match: { + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + } + }, + { + $project: { + _id: 1, + estimate_since_compact: 1, + compacted_state: 1 + } + } + ], + // This is used for the next query. + cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }] + } + } + ], + { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } + ) + .toArray(); + + const cursor = result?.cursor?.[0]; + if (cursor == null) { + break; + } + lastId = cursor._id; + + const mapped = (result?.buckets ?? []).map((bucketState) => { + // The numbers, specifically the bytes, could be a bigint. Convert to Number to allow calculating ratios. + // BigInt precision is not needed here since this is only an estimate. + const updatedCount = bucketState.estimate_since_compact?.count ?? 0; + const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount; + const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0); + const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes; + const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0; + const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0; + return { + bucket: bucketState._id.b, + definitionId: getDefinitionId(bucketState), + estimatedCount: totalCount, + dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes) + }; + }); + + yield mapped.filter( + (bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio + ); + } + } + + protected async dirtyBucketBatchForChecksumsForCollection( + collection: mongo.Collection, + filter: mongo.Filter, + getDefinitionId: (state: mongo.WithId) => BucketDefinitionId | null + ): Promise { + const dirtyBuckets = await collection + .find(filter, { + projection: { + _id: 1, + estimate_since_compact: 1, + compacted_state: 1 + }, + sort: { + 'estimate_since_compact.count': -1 + }, + limit: 200, + maxTimeMS: MONGO_OPERATION_TIMEOUT_MS + }) + .toArray(); + + return dirtyBuckets.map((bucket) => ({ + bucket: bucket._id.b, + definitionId: getDefinitionId(bucket), + estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) + })); + } + + public abstract dirtyBucketBatches(options: { + minBucketChanges: number; + minChangeRatio: number; + }): AsyncGenerator; + + public abstract dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise; + + protected async compactDirtyBuckets() { + for await (const buckets of this.dirtyBucketBatches({ minBucketChanges: this.minBucketChanges, minChangeRatio: this.minChangeRatio })) { @@ -128,8 +283,8 @@ export class MongoCompactor { continue; } - for (let { bucket } of buckets) { - await this.compactSingleBucketRetried(bucket); + for (const { bucket, definitionId } of buckets) { + await this.compactSingleBucketRetried(bucket, definitionId); } } } @@ -139,11 +294,11 @@ export class MongoCompactor { * * This covers against occasional network or other database errors during a long compact job. */ - private async compactSingleBucketRetried(bucket: string) { + protected async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) { let retryCount = 0; while (true) { try { - await this.compactSingleBucket(bucket); + await this.compactSingleBucket(bucket, definitionId); break; } catch (e) { if (retryCount < 3 && isMongoServerError(e)) { @@ -157,64 +312,64 @@ export class MongoCompactor { } } - private async compactSingleBucket(bucket: string) { + protected async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { const idLimitBytes = this.idLimitBytes; - - let currentState: CurrentBucketState = { + const bucketContext = await this.getBucketDataContext(bucket, definitionId); + if (bucketContext == null) { + return; + } + const currentState: CurrentBucketState = { bucket, + definitionId: bucketContext.key.definitionId, seen: new Map(), trackingSize: 0, lastNotPut: null, opsSincePut: 0, - checksum: 0, opCount: 0, opBytes: 0 }; - // Constant lower bound - const lowerBound: BucketDataKey = { - g: this.group_id, - b: bucket, - o: new mongo.MinKey() as any - }; - - // Upper bound is adjusted for each batch - let upperBound: BucketDataKey = { - g: this.group_id, - b: bucket, - o: new mongo.MaxKey() as any - }; + // Constant lower bound. + const lowerBound = bucketContext.minId; + // Upper bound is adjusted for each batch. + let upperBound = bucketContext.maxId; while (true) { this.signal?.throwIfAborted(); - // Query one batch at a time, to avoid cursor timeouts - const cursor = this.db.bucket_data.aggregate( - [ - { - $match: { - _id: { - $gte: lowerBound, - $lt: upperBound - } - } - }, - { $sort: { _id: -1 } }, - { $limit: this.moveBatchQueryLimit }, - { - $project: { - _id: 1, - op: 1, - table: 1, - row_id: 1, - source_table: 1, - source_key: 1, - checksum: 1, - size: { $bsonSize: '$$ROOT' } - } + // Query one batch at a time, to avoid cursor timeouts. + const pipeline = [ + { + $match: { + _id: { + $gte: lowerBound, + $lt: upperBound + }, + // Workaround for a clustered collection bug where the $lt operator may include upperBound. + // Technically only needed for storage V3. + // https://jira.mongodb.org/browse/SERVER-121822 + '_id.o': { $lt: upperBound.o } } - ], + }, + { $sort: { _id: -1 } }, + { $limit: this.moveBatchQueryLimit }, + { + $project: { + _id: 1, + op: 1, + table: 1, + row_id: 1, + source_table: 1, + source_key: 1, + checksum: 1, + size: { $bsonSize: '$$ROOT' } + } + } + ]; + + const cursor = bucketContext.collection.aggregate( + pipeline, { // batchSize is 1 more than limit to auto-close the cursor. // See https://github.com/mongodb/node-mongodb-native/pull/4580 @@ -223,18 +378,25 @@ export class MongoCompactor { ); // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns. // Instead, we load up to the limit. - const batch = await cursor.toArray(); + const rawBatch = await cursor.toArray(); + const batch = rawBatch.map((document) => { + const { size, ...rest } = document; + return { + doc: bucketContext.fromPersistedDocument(rest), + size + }; + }); if (batch.length == 0) { - // We've reached the end + // We've reached the end. break; } - // Set upperBound for the next batch - upperBound = batch[batch.length - 1]._id; + // Reuse the exact collection _id value from Mongo for the next bound. + upperBound = rawBatch[rawBatch.length - 1]._id; - for (let doc of batch) { - if (doc._id.o > this.maxOpId) { + for (const { doc, size } of batch) { + if (doc.o > this.maxOpId) { continue; } @@ -243,19 +405,17 @@ export class MongoCompactor { let isPersistentPut = doc.op == 'PUT'; - currentState.opBytes += Number(doc.size); + currentState.opBytes += Number(size); if (doc.op == 'REMOVE' || doc.op == 'PUT') { const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`; const targetOp = currentState.seen.get(key); if (targetOp) { - // Will convert to MOVE, so don't count as PUT + // Will convert to MOVE, so don't count as PUT. isPersistentPut = false; this.updates.push({ updateOne: { - filter: { - _id: doc._id - }, + filter: { _id: bucketContext.docId(doc.o) }, update: { $set: { op: 'MOVE', @@ -268,24 +428,20 @@ export class MongoCompactor { row_id: 1, data: 1 } - } + } satisfies mongo.UpdateFilter } }); - currentState.opBytes += 200 - Number(doc.size); // TODO: better estimate for this - } else { - if (currentState.trackingSize >= idLimitBytes) { - // Reached memory limit. - // Keep the highest seen values in this case. - } else { - // flatstr reduces the memory usage by flattening the string - currentState.seen.set(utils.flatstr(key), doc._id.o); - // length + 16 for the string - // 24 for the bigint - // 50 for map overhead - // 50 for additional overhead - currentState.trackingSize += key.length + 140; - } + // TODO: better estimate for this. + currentState.opBytes += 200 - Number(size); + } else if (currentState.trackingSize < idLimitBytes) { + // flatstr reduces the memory usage by flattening the string. + currentState.seen.set(utils.flatstr(key), doc.o); + // length + 16 for the string + // 24 for the bigint + // 50 for map overhead + // 50 for additional overhead + currentState.trackingSize += key.length + 140; } } @@ -294,41 +450,37 @@ export class MongoCompactor { currentState.opsSincePut = 0; } else if (doc.op != 'CLEAR') { if (currentState.lastNotPut == null) { - currentState.lastNotPut = doc._id.o; + currentState.lastNotPut = doc.o; } currentState.opsSincePut += 1; } if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) { - await this.flush(); + await this.flush(bucketContext); } } logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`); } - // Free memory before clearing bucket + // Free memory before clearing the bucket. currentState.seen.clear(); if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) { logger.info( `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` ); - // Need flush() before clear() - await this.flush(); - await this.clearBucket(currentState); + // Need flush() before clear(). + await this.flush(bucketContext); + await this.clearBucket(currentState, bucketContext); } - // Do this _after_ clearBucket so that we have accurate counts. + // Do this after clearBucket so we have accurate counts. this.updateBucketChecksums(currentState); - - // Need another flush after updateBucketChecksums() - await this.flush(); + // Need another flush after updateBucketChecksums(). + await this.flush(bucketContext); } - /** - * Call when done with a bucket. - */ - private updateBucketChecksums(state: CurrentBucketState) { + protected updateBucketChecksums(state: CurrentBucketState) { if (state.opCount < 0) { throw new ServiceAssertionError( `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}` @@ -336,12 +488,7 @@ export class MongoCompactor { } this.bucketStateUpdates.push({ updateOne: { - filter: { - _id: { - g: this.group_id, - b: state.bucket - } - }, + filter: this.bucketStateFilter(state.bucket, state.definitionId), update: { $set: { compacted_state: { @@ -351,14 +498,13 @@ export class MongoCompactor { bytes: state.opBytes }, estimate_since_compact: { - // Note: There could have been a whole bunch of new operations added to the bucket _while_ compacting, - // which we don't currently cater for. - // We could potentially query for that, but that could add overhead. + // There could have been a whole bunch of new operations added to the bucket while compacting, + // which we don't currently cater for. We could potentially query for that, but that adds overhead. count: 0, bytes: 0 } } - }, + } satisfies mongo.UpdateFilter, // We generally expect this to have been created before. // We don't create new ones here, to avoid issues with the unique index on bucket_updates. upsert: false @@ -366,23 +512,24 @@ export class MongoCompactor { }); } - private async flush() { + protected async flush(col: SingleBucketStore) { if (this.updates.length > 0) { logger.info(`Compacting ${this.updates.length} ops`); - await this.db.bucket_data.bulkWrite(this.updates, { - // Order is not important. - // Since checksums are not affected, these operations can happen in any order, - // and it's fine if the operations are partially applied. - // Each individual operation is atomic. + await col.collection.bulkWrite(this.updates, { + // Order is not important. Since checksums are not affected, these operations can happen in any order, + // and it's fine if the operations are partially applied. Each individual operation is atomic. ordered: false }); this.updates = []; } + + await this.flushBucketStateUpdates(); + } + + private async flushBucketStateUpdates() { if (this.bucketStateUpdates.length > 0) { logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`); - await this.db.bucket_state.bulkWrite(this.bucketStateUpdates, { - ordered: false - }); + await this.writeBucketStateUpdates(); this.bucketStateUpdates = []; } } @@ -390,26 +537,15 @@ export class MongoCompactor { /** * Perform a CLEAR compact for a bucket. * - * - * @param bucket bucket name - * @param op op_id of the last non-PUT operation, which will be converted to CLEAR. + * @param currentState tracks the last non-PUT op, which will be converted to CLEAR. */ - private async clearBucket(currentState: CurrentBucketState) { - const bucket = currentState.bucket; + protected async clearBucket(currentState: CurrentBucketState, col: SingleBucketStore) { const clearOp = currentState.lastNotPut!; const opFilter = { _id: { - $gte: { - g: this.group_id, - b: bucket, - o: new mongo.MinKey() as any - }, - $lte: { - g: this.group_id, - b: bucket, - o: clearOp - } + $gte: col.minId, + $lte: col.docId(clearOp) } }; @@ -424,39 +560,40 @@ export class MongoCompactor { // We need a transaction per batch to make sure checksums stay consistent. await session.withTransaction( async () => { - const query = this.db.bucket_data.find(opFilter, { - session, - sort: { _id: 1 }, - projection: { - _id: 1, - op: 1, - checksum: 1, - target_op: 1 - }, - limit: this.clearBatchLimit - }); + const query = col.collection.find>( + opFilter, + { + session, + sort: { _id: 1 }, + projection: { + _id: 1, + op: 1, + checksum: 1, + target_op: 1 + }, + limit: this.clearBatchLimit + } + ); let checksum = 0; - let lastOpId: BucketDataKey | null = null; + let lastOp: Pick | null = null; let targetOp: bigint | null = null; let gotAnOp = false; let numberOfOpsToClear = 0; - for await (let op of query.stream()) { + for await (const rawOp of query.stream()) { + const op = col.fromPartialPersistedDocument(rawOp); + if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') { checksum = utils.addChecksums(checksum, Number(op.checksum)); - lastOpId = op._id; + lastOp = op; numberOfOpsToClear += 1; if (op.op != 'CLEAR') { gotAnOp = true; } - if (op.target_op != null) { - if (targetOp == null || op.target_op > targetOp) { - targetOp = op.target_op; - } + if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) { + targetOp = op.target_op; } } else { - throw new ReplicationAssertionError( - `Unexpected ${op.op} operation at ${op._id.g}:${op._id.b}:${op._id.o}` - ); + throw new ReplicationAssertionError(`Unexpected ${op.op} operation at ${this.formatBucketDataKey(op)}`); } } if (!gotAnOp) { @@ -464,31 +601,25 @@ export class MongoCompactor { return; } - logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOpId?.o}`); - await this.db.bucket_data.deleteMany( + logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?.o}`); + await col.collection.deleteMany( { _id: { - $gte: { - g: this.group_id, - b: bucket, - o: new mongo.MinKey() as any - }, - $lte: lastOpId! + $gte: col.minId, + $lte: col.docId(lastOp!.o) } }, { session } ); - await this.db.bucket_data.insertOne( - { - _id: lastOpId!, - op: 'CLEAR', - checksum: BigInt(checksum), - data: null, - target_op: targetOp - }, - { session } - ); + const op = col.toPersistedDocument({ + o: lastOp!.o, + op: 'CLEAR', + checksum: BigInt(checksum), + data: null, + target_op: targetOp + }); + await col.collection.insertOne(op, { session }); opCountDiff = -numberOfOpsToClear + 1; }, @@ -497,7 +628,7 @@ export class MongoCompactor { readConcern: { level: 'snapshot' } } ); - // Update _outside_ the transaction, since the transaction can be retried multiple times. + // Update outside the transaction, since the transaction can be retried multiple times. currentState.opCount += opCountDiff; } } finally { @@ -505,211 +636,22 @@ export class MongoCompactor { } } - /** - * Subset of compact, only populating checksums where relevant. - */ - async populateChecksums(options: { minBucketChanges: number }): Promise { - let count = 0; - while (true) { - this.signal?.throwIfAborted(); - const buckets = await this.dirtyBucketBatchForChecksums(options); - if (buckets.length == 0) { - // All done - break; - } - this.signal?.throwIfAborted(); - - const start = Date.now(); - - // Filter batch by estimated bucket size, to reduce possibility of timeouts - let checkBuckets: typeof buckets = []; - let totalCountEstimate = 0; - for (let bucket of buckets) { - checkBuckets.push(bucket); - totalCountEstimate += bucket.estimatedCount; - if (totalCountEstimate > 50_000) { - break; - } - } - logger.info( - `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` - ); - await this.updateChecksumsBatch(checkBuckets.map((b) => b.bucket)); - logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); - count += checkBuckets.length; - } - return { buckets: count }; - } + protected async updateChecksumsBatch(buckets: Pick[]) { + const checksums = await this.computeChecksumsForBuckets(buckets); + const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId])); - /** - * Return batches of dirty buckets. - * - * Can be used to iterate through all buckets. - * - * minBucketChanges: minimum number of changes for a bucket to be included in the results. - * minChangeRatio: minimum ratio of changes to total ops for a bucket to be included in the results, number between 0 and 1. - */ - private async *dirtyBucketBatches(options: { - minBucketChanges: number; - minChangeRatio: number; - }): AsyncGenerator<{ bucket: string; estimatedCount: number }[]> { - // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only buckets with changes. - // This works well if there are only a small number of buckets with changes. - // However, if buckets are continuosly modified while we are compacting, we get the same buckets over and over again. - // This has caused the compact process to re-read the same collection around 5x times in total, which is very inefficient. - // To solve this, we now just iterate through all buckets, and filter out the ones with low changes. - - if (options.minBucketChanges <= 0) { - throw new ReplicationAssertionError('minBucketChanges must be >= 1'); - } - let lastId = { g: this.group_id, b: new mongo.MinKey() as any }; - const maxId = { g: this.group_id, b: new mongo.MaxKey() as any }; - while (true) { - // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, we use an aggregation pipeline - // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria, rather than limiting - // on the output number. - const [result] = await this.db.bucket_state - .aggregate<{ - buckets: Pick[]; - cursor: Pick[]; - }>( - [ - { - $match: { - _id: { $gt: lastId, $lt: maxId } - } - }, - { - $sort: { _id: 1 } - }, - { - // Scan a fixed number of docs each query so sparse matches don't block progress. - $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE - }, - { - $facet: { - // This is the results for the batch - buckets: [ - { - $match: { - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - } - }, - { - $project: { - _id: 1, - estimate_since_compact: 1, - compacted_state: 1 - } - } - ], - // This is used for the next query. - cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }] - } - } - ], - { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } - ) - .toArray(); - - const cursor = result?.cursor?.[0]; - if (cursor == null) { - break; - } - lastId = cursor._id; - - const mapped = (result?.buckets ?? []).map((b) => { - // The numbers, specifically the bytes, could be a bigint. We convert to Number to allow calculating the ratios. - // BigInt precision is not needed here since it's just an estimate. - const updatedCount = b.estimate_since_compact?.count ?? 0; - const totalCount = (b.compacted_state?.count ?? 0) + updatedCount; - const updatedBytes = Number(b.estimate_since_compact?.bytes ?? 0); - const totalBytes = Number(b.compacted_state?.bytes ?? 0) + updatedBytes; - const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0; - const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0; - return { - bucket: b._id.b, - estimatedCount: totalCount, - dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes) - }; - }); - const filtered = mapped.filter( - (b) => b.estimatedCount >= options.minBucketChanges && b.dirtyRatio >= options.minChangeRatio - ); - yield filtered; - } - } - - /** - * Returns a batch of dirty buckets - buckets with most changes first. - * - * This cannot be used to iterate on its own - the client is expected to process these buckets and - * set estimate_since_compact.count: 0 when done, before fetching the next batch. - * - * Unlike dirtyBucketBatches, used for compacting, this is specifically designed to be resuamble after a restart, - * since it is used as the last step for initial replication. - * - * We currently don't get new data while doing populateChecksums, so we don't need to worry about buckets changing while processing. - */ - private async dirtyBucketBatchForChecksums(options: { - minBucketChanges: number; - }): Promise<{ bucket: string; estimatedCount: number }[]> { - if (options.minBucketChanges <= 0) { - throw new ReplicationAssertionError('minBucketChanges must be >= 1'); - } - // We make use of an index on {_id.g: 1, 'estimate_since_compact.count': -1} - const dirtyBuckets = await this.db.bucket_state - .find( - { - '_id.g': this.group_id, - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - }, - { - projection: { - _id: 1, - estimate_since_compact: 1, - compacted_state: 1 - }, - sort: { - 'estimate_since_compact.count': -1 - }, - limit: 200, - maxTimeMS: MONGO_OPERATION_TIMEOUT_MS - } - ) - .toArray(); - - return dirtyBuckets.map((bucket) => ({ - bucket: bucket._id.b, - estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) - })); - } - - private async updateChecksumsBatch(buckets: string[]) { - const checksums = await this.storage.checksums.computePartialChecksumsDirect( - buckets.map((bucket) => { - return { - bucket, - source: {} as any, - end: this.maxOpId - }; - }) - ); - - for (let bucketChecksum of checksums.values()) { + for (const bucketChecksum of checksums.values()) { if (isPartialChecksum(bucketChecksum)) { - // Should never happen since we don't specify `start` + // Should never happen since we don't specify `start`. throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`); } this.bucketStateUpdates.push({ updateOne: { - filter: { - _id: { - g: this.group_id, - b: bucketChecksum.bucket - } - }, + filter: this.bucketStateFilter( + bucketChecksum.bucket, + definitionIdByBucket.get(bucketChecksum.bucket) ?? null + ), update: { $set: { compacted_state: { @@ -723,14 +665,34 @@ export class MongoCompactor { bytes: 0 } } - }, - // We don't create new ones here - it gets tricky to get the last_op right with the unique index on: - // bucket_updates: {'id.g': 1, 'last_op': 1} + } satisfies mongo.UpdateFilter, + // We don't create new ones here - it gets tricky to get the last_op right with the unique index on + // bucket_updates. upsert: false } }); } - await this.flush(); + await this.flushBucketStateUpdates(); + } + + protected formatBucketDataKey(doc: Pick) { + return `${doc.bucketKey.replicationStreamId}:${doc.bucketKey.bucket}:${doc.o}`; } + + protected abstract writeBucketStateUpdates(): Promise; + protected abstract computeChecksumsForBuckets( + buckets: Pick[] + ): Promise; + protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document; + + protected abstract getBucketDataContext( + bucket: string, + definitionId: BucketDefinitionId | null + ): Promise; +} + +export interface BucketDataCollectionContext { + bucketKey: BucketKey; + collection: mongo.Collection; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts index 066693fcf..c7bf5342a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts @@ -2,8 +2,14 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { logger } from '@powersync/lib-services-framework'; import { bson, CompactOptions, InternalOpId } from '@powersync/service-core'; import { LRUCache } from 'lru-cache'; -import { VersionedPowerSyncMongo } from './db.js'; -import { BucketParameterDocument } from './models.js'; +import type { VersionedPowerSyncMongo } from './db.js'; + +type ParameterCompactionReadDocument = { + _id: InternalOpId; + key: mongo.Document; + lookup: unknown; + bucket_parameters?: unknown[] | null; +}; /** * Compacts parameter lookup data (the bucket_parameters collection). @@ -12,16 +18,28 @@ import { BucketParameterDocument } from './models.js'; * * For background, see the `/docs/parameters-lookups.md` file. */ -export class MongoParameterCompactor { +export abstract class MongoParameterCompactor { constructor( - private db: VersionedPowerSyncMongo, - private group_id: number, - private checkpoint: InternalOpId, - private options: CompactOptions + protected readonly db: VersionedPowerSyncMongo, + protected readonly group_id: number, + protected readonly checkpoint: InternalOpId, + protected readonly options: CompactOptions ) {} async compact() { logger.info(`Compacting parameters for sync config ${this.group_id} up to checkpoint ${this.checkpoint}`); + for (const collection of await this.getCollections()) { + await this.compactCollection(collection); + } + } + + protected abstract getCollections(): Promise[]>; + + protected abstract collectionFilter(): mongo.Document; + + protected abstract deleteFilter(doc: mongo.Document): mongo.Document; + + protected async compactCollection(collection: mongo.Collection) { // This is the currently-active checkpoint. // We do not remove any data that may be used by this checkpoint. // snapshot queries ensure that if any clients are still using older checkpoints, they would @@ -32,43 +50,38 @@ export class MongoParameterCompactor { // In theory, we could let MongoDB do more of the work here, by grouping by (key, lookup) // in MongoDB already. However, that risks running into cases where MongoDB needs to process // very large amounts of data before returning results, which could lead to timeouts. - const cursor = this.db.bucket_parameters.find( - { - 'key.g': this.group_id - }, - { - sort: { lookup: 1, _id: 1 }, - batchSize: 10_000, - projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 } - } - ); + const cursor = collection.find(this.collectionFilter(), { + sort: { lookup: 1, _id: 1 }, + batchSize: 10_000, + projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 } + }); // The index doesn't cover sorting by key, so we keep our own cache of the last seen key. let lastByKey = new LRUCache({ max: this.options.compactParameterCacheLimit ?? 10_000 }); let removeIds: InternalOpId[] = []; - let removeDeleted: mongo.AnyBulkWriteOperation[] = []; + let removeDeleted: mongo.AnyBulkWriteOperation[] = []; let checkedEntries = 0; let checkedEntriesAtLastLog = 0; let lastProgressLogTime = Date.now(); const flush = async (force: boolean) => { if (removeIds.length >= 1000 || (force && removeIds.length > 0)) { - const results = await this.db.bucket_parameters.deleteMany({ _id: { $in: removeIds } }); + const results = await collection.deleteMany({ _id: { $in: removeIds } } as any); logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`); removeIds = []; } if (removeDeleted.length > 10 || (force && removeDeleted.length > 0)) { - const results = await this.db.bucket_parameters.bulkWrite(removeDeleted); + const results = await collection.bulkWrite(removeDeleted); logger.info(`Removed ${results.deletedCount} (${removeDeleted.length}) deleted parameter entries`); removeDeleted = []; } }; while (await cursor.hasNext()) { - const batch = cursor.readBufferedDocuments(); + const batch = cursor.readBufferedDocuments() as unknown as ParameterCompactionReadDocument[]; checkedEntries += batch.length; const now = Date.now(); if (now - lastProgressLogTime >= 60_000) { @@ -79,7 +92,7 @@ export class MongoParameterCompactor { checkedEntriesAtLastLog = checkedEntries; } - for (let doc of batch) { + for (const doc of batch) { if (doc._id >= checkpoint) { continue; } @@ -103,7 +116,7 @@ export class MongoParameterCompactor { // in the cache due to cache size limits. So we need to explicitly remove all earlier operations. removeDeleted.push({ deleteMany: { - filter: { 'key.g': doc.key.g, lookup: doc.lookup, _id: { $lte: doc._id }, key: doc.key } + filter: this.deleteFilter(doc) } }); } @@ -113,6 +126,6 @@ export class MongoParameterCompactor { } await flush(true); - logger.info('Parameter compaction completed'); + logger.info(`Parameter compaction completed for ${collection.collectionName}`); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts new file mode 100644 index 000000000..94da94c3b --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts @@ -0,0 +1,76 @@ +import { ServiceAssertionError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { + BucketDataScope, + BucketDataSource, + CompatibilityOption, + DEFAULT_HYDRATION_STATE, + HydratedSyncRules, + HydrationState, + ParameterIndexLookupCreator, + SyncConfigWithErrors, + versionedHydrationState +} from '@powersync/service-sync-rules'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { StorageConfig } from './models.js'; + +export class MongoPersistedSyncRules implements storage.PersistedSyncRules { + public readonly hydrationState: HydrationState; + + constructor( + public readonly id: number, + public readonly sync_rules: SyncConfigWithErrors, + public readonly slot_name: string, + private readonly mapping: BucketDefinitionMapping | null, + private readonly storageConfig: StorageConfig + ) { + if (this.storageConfig.incrementalReprocessing) { + if (this.mapping == null) { + throw new ServiceAssertionError(`mapping is required for v3 storage`); + } + this.hydrationState = new MongoHydrationState(this.mapping, this.id); + } else if ( + !this.sync_rules.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) && + !this.storageConfig.versionedBuckets + ) { + this.hydrationState = DEFAULT_HYDRATION_STATE; + } else { + this.hydrationState = versionedHydrationState(this.id); + } + } + + hydratedSyncRules(): HydratedSyncRules { + return this.sync_rules.config.hydrate({ hydrationState: this.hydrationState }); + } +} + +class MongoHydrationState implements HydrationState { + constructor( + private readonly mapping: BucketDefinitionMapping, + private readonly version: number + ) {} + + getBucketSourceScope(source: BucketDataSource): BucketDataScope { + // Keep this aligned with versionedHydrationState() for now. + // + // Previous Mongo-specific behavior: + // const defId = this.mapping.bucketSourceId(source); + // return { + // bucketPrefix: defId, + // source + // }; + return { + bucketPrefix: `${this.version}#${source.uniqueName}`, + source + }; + } + + getParameterIndexLookupScope(source: ParameterIndexLookupCreator) { + const defId = this.mapping.parameterLookupId(source); + return { + lookupName: defId, + queryId: '', + source + }; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts index b7fad7863..f6baf7bab 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts @@ -1,11 +1,14 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js'; import { MongoSyncRulesLock } from './MongoSyncRulesLock.js'; import { PowerSyncMongo } from './db.js'; import { getMongoStorageConfig, SyncRuleDocument } from './models.js'; export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesContent { public current_lock: MongoSyncRulesLock | null = null; + public readonly mapping: BucketDefinitionMapping; constructor( private db: PowerSyncMongo, @@ -25,12 +28,26 @@ export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesCo active: doc.state == 'ACTIVE', storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION }); + this.mapping = BucketDefinitionMapping.fromSyncRules(doc); } getStorageConfig() { return getMongoStorageConfig(this.storageVersion); } + parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules { + const parsed = super.parsed(options); + const storageConfig = this.getStorageConfig(); + + return new MongoPersistedSyncRules( + parsed.id, + parsed.sync_rules, + parsed.slot_name, + storageConfig.incrementalReprocessing ? this.mapping : null, + storageConfig + ); + } + async lock() { const lock = await MongoSyncRulesLock.createLock(this.db.versioned(this.getStorageConfig()), this); this.current_lock = lock; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 67b975376..59021f9c0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -11,48 +11,43 @@ import { BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, CheckpointChanges, - deserializeParameterLookup, GetCheckpointChangesOptions, InternalOpId, - internalToExternalOpId, maxLsn, mergeAsyncIterables, PopulateChecksumCacheOptions, PopulateChecksumCacheResults, - ProtocolOpId, ReplicationCheckpoint, storage, utils, WatchWriteCheckpointOptions } from '@powersync/service-core'; -import { JSONBig } from '@powersync/service-jsonbig'; import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { LRUCache } from 'lru-cache'; import * as timers from 'timers/promises'; -import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js'; +import { retryOnMongoMaxTimeMSExpired } from '../../utils/util.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; -import { VersionedPowerSyncMongo } from './db.js'; -import { - BucketDataDocument, - BucketDataKey, - BucketStateDocument, - SourceKey, - SourceTableDocument, - StorageConfig -} from './models.js'; -import { MongoBucketBatch } from './MongoBucketBatch.js'; +import { MongoSyncBucketStorageContext } from './common/MongoSyncBucketStorageContext.js'; +import type { VersionedPowerSyncMongo } from './db.js'; +import { CommonSourceTableDocument, StorageConfig } from './models.js'; +import { MongoBucketBatchOptions } from './MongoBucketBatch.js'; import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; -import { MongoCompactor } from './MongoCompactor.js'; +import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js'; import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; export interface MongoSyncBucketStorageOptions { - checksumOptions?: Omit; + checksumOptions?: Omit; storageConfig: StorageConfig; } +interface InternalCheckpointChanges extends CheckpointChanges { + updatedWriteCheckpoints: Map; + invalidateWriteCheckpoints: boolean; +} + /** * Only keep checkpoints around for a minute, before fetching a fresh one. * @@ -64,32 +59,30 @@ export interface MongoSyncBucketStorageOptions { */ const CHECKPOINT_TIMEOUT_MS = 60_000; -export class MongoSyncBucketStorage +export abstract class MongoSyncBucketStorage extends BaseObserver implements storage.SyncRulesBucketStorage { + readonly db: VersionedPowerSyncMongo; [DO_NOT_LOG] = true; - private readonly db: VersionedPowerSyncMongo; readonly checksums: MongoChecksums; private parsedSyncRulesCache: { parsed: HydratedSyncRules; options: storage.ParseSyncRulesOptions } | undefined; private writeCheckpointAPI: MongoWriteCheckpointAPI; + #storageInitialized = false; constructor( public readonly factory: MongoBucketStorage, public readonly group_id: number, - private readonly sync_rules: MongoPersistedSyncRulesContent, + protected readonly sync_rules: MongoPersistedSyncRulesContent, public readonly slot_name: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions ) { super(); this.db = factory.db.versioned(sync_rules.getStorageConfig()); - this.checksums = new MongoChecksums(this.db, this.group_id, { - ...options.checksumOptions, - storageConfig: options?.storageConfig - }); + this.checksums = this.createMongoChecksums(options); this.writeCheckpointAPI = new MongoWriteCheckpointAPI({ db: this.db, mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED, @@ -97,10 +90,35 @@ export class MongoSyncBucketStorage }); } + /** + * Not for external use - public here for tests only. + * + * @internal + */ + abstract createMongoCompactor(options: MongoCompactOptions): MongoCompactor; + + protected abstract createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums; + protected abstract createMongoParameterCompactor( + checkpoint: InternalOpId, + options: storage.CompactOptions + ): MongoParameterCompactor; + get writeCheckpointMode() { return this.writeCheckpointAPI.writeCheckpointMode; } + get mapping() { + return this.sync_rules.mapping; + } + + protected get versionContext(): MongoSyncBucketStorageContext { + return { + db: this.db, + group_id: this.group_id, + mapping: this.mapping + }; + } + setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { this.writeCheckpointAPI.setWriteCheckpointMode(mode); } @@ -118,10 +136,6 @@ export class MongoSyncBucketStorage getParsedSyncRules(options: storage.ParseSyncRulesOptions): HydratedSyncRules { const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; - /** - * Check if the cached sync rules, if present, had the same options. - * Parse sync rules if the options are different or if there is no cached value. - */ if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncRules(), options }; } @@ -143,26 +157,15 @@ export class MongoSyncBucketStorage } ); if (!doc?.snapshot_done || !['ACTIVE', 'ERRORED'].includes(doc.state)) { - // Sync rules not active - return null return null; } - // Specifically using operationTime instead of clusterTime - // There are 3 fields in the response: - // 1. operationTime, not exposed for snapshot sessions (used for causal consistency) - // 2. clusterTime (used for connection management) - // 3. atClusterTime, which is session.snapshotTime - // We use atClusterTime, to match the driver's internal snapshot handling. - // There are cases where clusterTime > operationTime and atClusterTime, - // which could cause snapshot queries using this as the snapshotTime to timeout. - // This was specifically observed on MongoDB 6.0 and 7.0. const snapshotTime = (session as any).snapshotTime as bson.Timestamp | undefined; if (snapshotTime == null) { throw new ServiceAssertionError('Missing snapshotTime in getCheckpoint()'); } return new MongoReplicationCheckpoint( this, - // null/0n is a valid checkpoint in some cases, for example if the initial snapshot was empty doc.last_checkpoint ?? 0n, doc.last_checkpoint_lsn ?? null, snapshotTime @@ -170,7 +173,23 @@ export class MongoSyncBucketStorage }); } + protected abstract initializeVersionStorage(): Promise; + + private async initializeStorage() { + if (this.#storageInitialized) { + return; + } + + await this.db.initializeStreamStorage(this.group_id); + await this.initializeVersionStorage(); + this.#storageInitialized = true; + } + + protected abstract createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch; + async createWriter(options: storage.CreateWriterOptions): Promise { + await this.initializeStorage(); + const doc = await this.db.sync_rules.findOne( { _id: this.group_id @@ -179,10 +198,11 @@ export class MongoSyncBucketStorage ); const checkpoint_lsn = doc?.last_checkpoint_lsn ?? null; - const writer = new MongoBucketBatch({ + const batchOptions = { logger: options.logger, db: this.db, syncRules: this.sync_rules.parsed(options).hydratedSyncRules(), + mapping: this.sync_rules.mapping, groupId: this.group_id, slotName: this.slot_name, lastCheckpointLsn: checkpoint_lsn, @@ -191,14 +211,12 @@ export class MongoSyncBucketStorage storeCurrentData: options.storeCurrentData, skipExistingRows: options.skipExistingRows ?? false, markRecordUnavailable: options.markRecordUnavailable - }); + }; + const writer = this.createWriterImpl(batchOptions); this.iterateListeners((cb) => cb.batchStarted?.(writer)); return writer; } - /** - * @deprecated Use `createWriter()` with `await using` instead. - */ async startBatch( options: storage.CreateWriterOptions, callback: (batch: storage.BucketStorageBatch) => Promise @@ -209,6 +227,16 @@ export class MongoSyncBucketStorage return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; } + protected abstract sourceTableBaseId(): Partial; + + protected abstract augmentCreatedSourceTableDocument( + createDoc: CommonSourceTableDocument, + options: storage.ResolveTableOptions, + candidateSourceTable: storage.SourceTable + ): void; + + protected abstract initializeResolvedSourceRecords(sourceTableId: bson.ObjectId): Promise; + async resolveTable(options: storage.ResolveTableOptions): Promise { const { group_id, connection_id, connection_tag, entity_descriptor } = options; @@ -220,23 +248,36 @@ export class MongoSyncBucketStorage type_oid: column.typeId })); let result: storage.ResolveTableResult | null = null; + let initializeSourceRecordsFor: bson.ObjectId | null = null; + + const baseId = this.sourceTableBaseId(); await this.db.client.withSession(async (session) => { - const col = this.db.source_tables; - let filter: Partial = { - group_id: group_id, + const col = this.db.commonSourceTables(group_id); + let filter: Partial = { + ...baseId, connection_id: connection_id, schema_name: schema, table_name: name, replica_id_columns2: normalizedReplicaIdColumns }; + if (objectId != null) { filter.relation_id = objectId; } let doc = await col.findOne(filter, { session }); if (doc == null) { - doc = { - _id: new bson.ObjectId(), - group_id: group_id, + const candidateSourceTable = new storage.SourceTable({ + id: new bson.ObjectId(), + connectionTag: connection_tag, + objectId: objectId, + schema: schema, + name: name, + replicaIdColumns: replicaIdColumns, + snapshotComplete: false + }); + const createDoc: CommonSourceTableDocument = { + _id: candidateSourceTable.id as bson.ObjectId, + ...(baseId as any), connection_id: connection_id, relation_id: objectId, schema_name: schema, @@ -246,8 +287,11 @@ export class MongoSyncBucketStorage snapshot_done: false, snapshot_status: undefined }; + this.augmentCreatedSourceTableDocument(createDoc, options, candidateSourceTable); + doc = createDoc; await col.insertOne(doc, { session }); + initializeSourceRecordsFor = doc._id; } const sourceTable = new storage.SourceTable({ id: doc._id, @@ -271,16 +315,14 @@ export class MongoSyncBucketStorage }; let dropTables: storage.SourceTable[] = []; - // Detect tables that are either renamed, or have different replica_id_columns let truncateFilter = [{ schema_name: schema, table_name: name }] as any[]; if (objectId != null) { - // Only detect renames if the source uses relation ids. truncateFilter.push({ relation_id: objectId }); } const truncate = await col .find( { - group_id: group_id, + ...baseId, connection_id: connection_id, _id: { $ne: doc._id }, $or: truncateFilter @@ -307,223 +349,36 @@ export class MongoSyncBucketStorage dropTables: dropTables }; }); + if (initializeSourceRecordsFor != null) { + await this.initializeResolvedSourceRecords(initializeSourceRecordsFor); + } return result!; } + protected abstract getParameterSetsImpl( + checkpoint: MongoReplicationCheckpoint, + lookups: ScopedParameterLookup[] + ): Promise; + async getParameterSets( checkpoint: MongoReplicationCheckpoint, lookups: ScopedParameterLookup[] ): Promise { - return this.db.client.withSession({ snapshot: true }, async (session) => { - // Set the session's snapshot time to the checkpoint's snapshot time. - // An alternative would be to create the session when the checkpoint is created, but managing - // the session lifetime would become more complex. - // Starting and ending sessions are cheap (synchronous when no transactions are used), - // so this should be fine. - // This is a roundabout way of setting {readConcern: {atClusterTime: clusterTime}}, since - // that is not exposed directly by the driver. - // Future versions of the driver may change the snapshotTime behavior, so we need tests to - // validate that this works as expected. We test this in the compacting tests. - setSessionSnapshotTime(session, checkpoint.snapshotTime); - const lookupFilter = lookups.map((lookup) => { - return storage.serializeLookup(lookup); - }); - // This query does not use indexes super efficiently, apart from the lookup filter. - // From some experimentation I could do individual lookups more efficient using an index - // on {'key.g': 1, lookup: 1, 'key.t': 1, 'key.k': 1, _id: -1}, - // but could not do the same using $group. - // For now, just rely on compacting to remove extraneous data. - // For a description of the data format, see the `/docs/parameters-lookups.md` file. - const rows = await this.db.bucket_parameters - .aggregate( - [ - { - $match: { - 'key.g': this.group_id, - lookup: { $in: lookupFilter }, - _id: { $lte: checkpoint.checkpoint } - } - }, - { - $sort: { - _id: -1 - } - }, - { - $group: { - _id: { key: '$key', lookup: '$lookup' }, - bucket_parameters: { - $first: '$bucket_parameters' - } - } - } - ], - { - session, - readConcern: 'snapshot', - // Limit the time for the operation to complete, to avoid getting connection timeouts - maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS - } - ) - .toArray() - .catch((e) => { - throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries'); - }); - const groupedParameters = rows.map((row) => { - return row.bucket_parameters; - }); - return groupedParameters.flat(); - }); + return this.getParameterSetsImpl(checkpoint, lookups); } + protected abstract getBucketDataBatchImpl( + checkpoint: utils.InternalOpId, + dataBuckets: storage.BucketDataRequest[], + options?: storage.BucketDataBatchOptions + ): AsyncIterable; + async *getBucketDataBatch( checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions ): AsyncIterable { - if (dataBuckets.length == 0) { - return; - } - let filters: mongo.Filter[] = []; - const bucketMap = new Map(dataBuckets.map((request) => [request.bucket, request.start])); - - if (checkpoint == null) { - throw new ServiceAssertionError('checkpoint is null'); - } - const end = checkpoint; - for (let { bucket: name, start } of dataBuckets) { - filters.push({ - _id: { - $gt: { - g: this.group_id, - b: name, - o: start - }, - $lte: { - g: this.group_id, - b: name, - o: end as any - } - } - }); - } - - // Internal naming: - // We do a query for one "batch", which may consist of multiple "chunks". - // Each chunk is limited to single bucket, and is limited in length and size. - // There are also overall batch length and size limits. - - const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; - const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; - - const cursor = this.db.bucket_data.find( - { - $or: filters - }, - { - session: undefined, - sort: { _id: 1 }, - limit: batchLimit, - // Increase batch size above the default 101, so that we can fill an entire batch in - // one go. - // batchSize is 1 more than limit to auto-close the cursor. - // See https://github.com/mongodb/node-mongodb-native/pull/4580 - batchSize: batchLimit + 1, - // Raw mode is returns an array of Buffer instead of parsed documents. - // We use it so that: - // 1. We can calculate the document size accurately without serializing again. - // 2. We can delay parsing the results until it's needed. - // We manually use bson.deserialize below - raw: true, - - // Limit the time for the operation to complete, to avoid getting connection timeouts - maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS - } - ) as unknown as mongo.FindCursor; - - // We want to limit results to a single batch to avoid high memory usage. - // This approach uses MongoDB's batch limits to limit the data here, which limits - // to the lower of the batch count and size limits. - // This is similar to using `singleBatch: true` in the find options, but allows - // detecting "hasMore". - let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => { - throw lib_mongo.mapQueryError(e, 'while reading bucket data'); - }); - if (data.length == batchLimit) { - // Limit reached - could have more data, despite the cursor being drained. - batchHasMore = true; - } - - let chunkSizeBytes = 0; - let currentChunk: utils.SyncBucketData | null = null; - let targetOp: InternalOpId | null = null; - - // Ordered by _id, meaning buckets are grouped together - for (let rawData of data) { - const row = bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocument; - const bucket = row._id.b; - - if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) { - // We need to start a new chunk - let start: ProtocolOpId | undefined = undefined; - if (currentChunk != null) { - // There is an existing chunk we need to yield - if (currentChunk.bucket == bucket) { - // Current and new chunk have the same bucket, so need has_more on the current one. - // If currentChunk.bucket != bucket, then we reached the end of the previous bucket, - // and has_more = false in that case. - currentChunk.has_more = true; - start = currentChunk.next_after; - } - - const yieldChunk = currentChunk; - currentChunk = null; - chunkSizeBytes = 0; - yield { chunkData: yieldChunk, targetOp: targetOp }; - targetOp = null; - } - - if (start == null) { - const startOpId = bucketMap.get(bucket); - if (startOpId == null) { - throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`); - } - start = internalToExternalOpId(startOpId); - } - currentChunk = { - bucket, - after: start, - has_more: false, - data: [], - next_after: start - }; - targetOp = null; - } - - const entry = mapOpEntry(row); - - if (row.target_op != null) { - // MOVE, CLEAR - if (targetOp == null || row.target_op > targetOp) { - targetOp = row.target_op; - } - } - - currentChunk.data.push(entry); - currentChunk.next_after = entry.op_id; - - chunkSizeBytes += rawData.byteLength; - } - - if (currentChunk != null) { - const yieldChunk = currentChunk; - currentChunk = null; - // This is the final chunk in the batch. - // There may be more data if and only if the batch we retrieved isn't complete. - yieldChunk.has_more = batchHasMore; - yield { chunkData: yieldChunk, targetOp: targetOp }; - targetOp = null; - } + yield* this.getBucketDataBatchImpl(checkpoint, dataBuckets, options); } async getChecksums( @@ -538,7 +393,6 @@ export class MongoSyncBucketStorage } async terminate(options?: storage.TerminateOptions) { - // Default is to clear the storage except when explicitly requested not to. if (!options || options?.clearStorage) { await this.clear(options); } @@ -583,32 +437,22 @@ export class MongoSyncBucketStorage }; } + protected abstract clearBucketData(signal?: AbortSignal): Promise; + + protected abstract clearParameterIndexes(signal?: AbortSignal): Promise; + + protected abstract clearSourceRecords(signal?: AbortSignal): Promise; + + protected abstract clearBucketState(signal?: AbortSignal): Promise; + + protected abstract clearSourceTables(signal?: AbortSignal): Promise; + async clear(options?: storage.ClearStorageOptions): Promise { - while (true) { - if (options?.signal?.aborted) { - throw new ReplicationAbortedError('Aborted clearing data', options.signal.reason); - } - try { - await this.clearIteration(); + const signal = options?.signal; - logger.info(`${this.slot_name} Done clearing data`); - return; - } catch (e: unknown) { - if (lib_mongo.isMongoServerError(e) && e.codeName == 'MaxTimeMSExpired') { - logger.info( - `${this.slot_name} Cleared batch of data in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...` - ); - await timers.setTimeout(lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5); - } else { - throw e; - } - } + if (signal?.aborted) { + throw new ReplicationAbortedError('Aborted clearing data', signal.reason); } - } - - private async clearIteration(): Promise { - // Individual operations here may time out with the maxTimeMS option. - // It is expected to still make progress, and continue on the next try. await this.db.sync_rules.updateOne( { @@ -628,39 +472,31 @@ export class MongoSyncBucketStorage }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } ); - await this.db.bucket_data.deleteMany( - { - _id: idPrefixFilter({ g: this.group_id }, ['b', 'o']) - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ); - await this.db.bucket_parameters.deleteMany( - { - 'key.g': this.group_id - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ); - - await this.db.common_current_data.deleteMany( - { - _id: idPrefixFilter({ g: this.group_id }, ['t', 'k']) - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ); - - await this.db.bucket_state.deleteMany( - { - _id: idPrefixFilter({ g: this.group_id }, ['b']) - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ); - await this.db.source_tables.deleteMany( - { - group_id: this.group_id - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ); + await this.clearBucketData(signal); + await this.clearParameterIndexes(signal); + await this.clearSourceRecords(signal); + await this.clearBucketState(signal); + await this.clearSourceTables(signal); + + this.#storageInitialized = false; + } + + protected async clearDeleteMany( + label: string, + operation: () => Promise, + signal?: AbortSignal + ): Promise { + await retryOnMongoMaxTimeMSExpired(operation, { + signal, + abortMessage: 'Aborted clearing data', + retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5, + onRetry: () => { + logger.info( + `${this.slot_name} Cleared batch of ${label} in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...` + ); + } + }); } async reportError(e: any): Promise { @@ -684,27 +520,22 @@ export class MongoSyncBucketStorage const checkpoint = await this.getCheckpointInternal(); maxOpId = checkpoint?.checkpoint ?? undefined; } - await new MongoCompactor(this, this.db, { ...options, maxOpId }).compact(); + await this.createMongoCompactor({ ...options, maxOpId }).compact(); if (maxOpId != null && options?.compactParameterData) { - await new MongoParameterCompactor(this.db, this.group_id, maxOpId, options).compact(); + await this.createMongoParameterCompactor(maxOpId, options).compact(); } } async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise { logger.info(`Populating persistent checksum cache...`); const start = Date.now(); - // We do a minimal compact here. - // We can optimize this in the future. - const compactor = new MongoCompactor(this, this.db, { + const compactor = this.createMongoCompactor({ ...options, - // Don't track updates for MOVE compacting memoryLimitMB: 0 }); const result = await compactor.populateChecksums({ - // There are cases with millions of small buckets, in which case it can take very long to - // populate the checksums, with minimal benefit. We skip the small buckets here. minBucketChanges: options.minBucketChanges ?? 10 }); const duration = Date.now() - start; @@ -712,72 +543,44 @@ export class MongoSyncBucketStorage return result; } - /** - * Instance-wide watch on the latest available checkpoint (op_id + lsn). - */ private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable { if (signal.aborted) { return; } - // If the stream is idle, we wait a max of a minute (CHECKPOINT_TIMEOUT_MS) before we get another checkpoint, - // to avoid stale checkpoint snapshots. This is what checkpointTimeoutStream() is for. - // Essentially, even if there are no actual checkpoint changes, we want a new snapshotTime every minute or so, - // to ensure that any new clients connecting will get a valid snapshotTime. const stream = mergeAsyncIterables( [this.checkpointChangesStream(signal), this.checkpointTimeoutStream(signal)], signal ); - // We only watch changes to the active sync rules. - // If it changes to inactive, we abort and restart with the new sync rules. for await (const _ of stream) { if (signal.aborted) { - // Would likely have been caught by the signal on the timeout or the upstream stream, but we check here anyway break; } const op = await this.getCheckpointInternal(); if (op == null) { - // Sync rules have changed - abort and restart. - // We do a soft close of the stream here - no error break; } - // Previously, we only yielded when the checkpoint or lsn changed. - // However, we always want to use the latest snapshotTime, so we skip that filtering here. - // That filtering could be added in the per-user streams if needed, but in general the capped collection - // should already only contain useful changes in most cases. yield op; } } - // Nothing is done here until a subscriber starts to iterate private readonly sharedIter = new BroadcastIterable((signal) => { return this.watchActiveCheckpoint(signal); }); - /** - * User-specific watch on the latest checkpoint and/or write checkpoint. - */ async *watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable { let lastCheckpoint: ReplicationCheckpoint | null = null; const iter = this.sharedIter[Symbol.asyncIterator](options.signal); let writeCheckpoint: bigint | null = null; - // true if we queried the initial write checkpoint, even if it doesn't exist let queriedInitialWriteCheckpoint = false; for await (const nextCheckpoint of iter) { - // lsn changes are not important by itself. - // What is important is: - // 1. checkpoint (op_id) changes. - // 2. write checkpoint changes for the specific user - if (nextCheckpoint.lsn != null && !queriedInitialWriteCheckpoint) { - // Lookup the first write checkpoint for the user when we can. - // There will not actually be one in all cases. writeCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({ sync_rules_id: this.group_id, user_id: options.user_id, @@ -793,15 +596,11 @@ export class MongoSyncBucketStorage lastCheckpoint.checkpoint == nextCheckpoint.checkpoint && lastCheckpoint.lsn == nextCheckpoint.lsn ) { - // No change - wait for next one - // In some cases, many LSNs may be produced in a short time. - // Add a delay to throttle the loop a bit. await timers.setTimeout(20 + 10 * Math.random()); continue; } if (lastCheckpoint == null) { - // First message for this stream - "INVALIDATE_ALL" means it will lookup all data yield { base: nextCheckpoint, writeCheckpoint, @@ -815,8 +614,6 @@ export class MongoSyncBucketStorage let updatedWriteCheckpoint = updates.updatedWriteCheckpoints.get(options.user_id) ?? null; if (updates.invalidateWriteCheckpoints) { - // Invalidated means there were too many updates to track the individual ones, - // so we switch to "polling" (querying directly in each stream). updatedWriteCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({ sync_rules_id: this.group_id, user_id: options.user_id, @@ -827,8 +624,6 @@ export class MongoSyncBucketStorage } if (updatedWriteCheckpoint != null && (writeCheckpoint == null || updatedWriteCheckpoint > writeCheckpoint)) { writeCheckpoint = updatedWriteCheckpoint; - // If it happened that we haven't queried a write checkpoint at this point, - // then we don't need to anymore, since we got an updated one. queriedInitialWriteCheckpoint = true; } @@ -848,12 +643,6 @@ export class MongoSyncBucketStorage } } - /** - * This watches the checkpoint_events capped collection for new documents inserted, - * and yields whenever one or more documents are inserted. - * - * The actual checkpoint must be queried on the sync_rules collection after this. - */ private async *checkpointChangesStream(signal: AbortSignal): AsyncGenerator { if (signal.aborted) { return; @@ -872,17 +661,13 @@ export class MongoSyncBucketStorage cursor.close().catch(() => {}); }); - // Yield once on start, regardless of whether there are documents in the cursor. - // This is to ensure that the first iteration of the generator yields immediately. yield; try { while (!signal.aborted) { const doc = await cursor.tryNext().catch((e) => { if (lib_mongo.isMongoServerError(e) && e.codeName === 'CappedPositionLost') { - // Cursor position lost, potentially due to a high rate of notifications cursor = query(); - // Treat as an event found, before querying the new cursor again return {}; } else { return Promise.reject(e); @@ -891,8 +676,6 @@ export class MongoSyncBucketStorage if (cursor.closed) { return; } - // Skip buffered documents, if any. We don't care about the contents, - // we only want to know when new documents are inserted. cursor.readBufferedDocuments(); if (doc != null) { yield; @@ -914,7 +697,6 @@ export class MongoSyncBucketStorage await timers.setTimeout(CHECKPOINT_TIMEOUT_MS, undefined, { signal }); } catch (e) { if (e.name == 'AbortError') { - // This is how we typically abort this stream, when all listeners are done return; } throw e; @@ -926,94 +708,37 @@ export class MongoSyncBucketStorage } } + protected abstract getDataBucketChangesImpl( + options: GetCheckpointChangesOptions + ): Promise>; + private async getDataBucketChanges( options: GetCheckpointChangesOptions ): Promise> { - const limit = 1000; - const bucketStateUpdates = await this.db.bucket_state - .find( - { - // We have an index on (_id.g, last_op). - '_id.g': this.group_id, - last_op: { $gt: options.lastCheckpoint.checkpoint } - }, - { - projection: { - '_id.b': 1 - }, - limit: limit + 1, - // batchSize is 1 more than limit to auto-close the cursor. - // See https://github.com/mongodb/node-mongodb-native/pull/4580 - batchSize: limit + 2, - singleBatch: true - } - ) - .toArray(); - - const buckets = bucketStateUpdates.map((doc) => doc._id.b); - const invalidateDataBuckets = buckets.length > limit; - - return { - invalidateDataBuckets: invalidateDataBuckets, - updatedDataBuckets: invalidateDataBuckets ? new Set() : new Set(buckets) - }; + return this.getDataBucketChangesImpl(options); } + protected abstract getParameterBucketChangesImpl( + options: GetCheckpointChangesOptions + ): Promise>; + private async getParameterBucketChanges( options: GetCheckpointChangesOptions ): Promise> { - const limit = 1000; - const parameterUpdates = await this.db.bucket_parameters - .find( - { - _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint }, - 'key.g': this.group_id - }, - { - projection: { - lookup: 1 - }, - limit: limit + 1, - // batchSize is 1 more than limit to auto-close the cursor. - // See https://github.com/mongodb/node-mongodb-native/pull/4580 - batchSize: limit + 2, - singleBatch: true - } - ) - .toArray(); - const invalidateParameterUpdates = parameterUpdates.length > limit; - - return { - invalidateParameterBuckets: invalidateParameterUpdates, - updatedParameterLookups: invalidateParameterUpdates - ? new Set() - : new Set(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookup(p.lookup)))) - }; + return this.getParameterBucketChangesImpl(options); } - // If we processed all connections together for each checkpoint, we could do a single lookup for all connections. - // In practice, specific connections may fall behind. So instead, we just cache the results of each specific lookup. - // TODO (later): - // We can optimize this by implementing it like ChecksumCache: We can use partial cache results to do - // more efficient lookups in some cases. private checkpointChangesCache = new LRUCache< string, InternalCheckpointChanges, { options: GetCheckpointChangesOptions } >({ - // Limit to 50 cache entries, or 10MB, whichever comes first. - // Some rough calculations: - // If we process 10 checkpoints per second, and a connection may be 2 seconds behind, we could have - // up to 20 relevant checkpoints. That gives us 20*20 = 400 potentially-relevant cache entries. - // That is a worst-case scenario, so we don't actually store that many. In real life, the cache keys - // would likely be clustered around a few values, rather than spread over all 400 potential values. max: 50, maxSize: 12 * 1024 * 1024, sizeCalculation: (value: InternalCheckpointChanges) => { - // Estimate of memory usage const paramSize = [...value.updatedParameterLookups].reduce((a, b) => a + b.length, 0); const bucketSize = [...value.updatedDataBuckets].reduce((a, b) => a + b.length, 0); - const writeCheckpointSize = value.updatedWriteCheckpoints.size * 30; // estiamte for user_id + bigint + const writeCheckpointSize = value.updatedWriteCheckpoints.size * 30; return 100 + paramSize + bucketSize + writeCheckpointSize; }, fetchMethod: async (_key, _staleValue, options) => { @@ -1040,11 +765,6 @@ export class MongoSyncBucketStorage } } -interface InternalCheckpointChanges extends CheckpointChanges { - updatedWriteCheckpoints: Map; - invalidateWriteCheckpoints: boolean; -} - class MongoReplicationCheckpoint implements ReplicationCheckpoint { #storage: MongoSyncBucketStorage; @@ -1066,7 +786,7 @@ class EmptyReplicationCheckpoint implements ReplicationCheckpoint { readonly checkpoint: InternalOpId = 0n; readonly lsn: string | null = null; - async getParameterSets(lookups: ScopedParameterLookup[]): Promise { + async getParameterSets(_lookups: ScopedParameterLookup[]): Promise { return []; } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/OperationBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/OperationBatch.ts index 95193042f..33d5209d2 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/OperationBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/OperationBatch.ts @@ -2,7 +2,7 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { storage } from '@powersync/service-core'; -import { mongoTableId } from '../storage-index.js'; +import { mongoTableId } from '../../utils/util.js'; /** * Maximum number of operations in a batch. diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts deleted file mode 100644 index 6971a59a2..000000000 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ /dev/null @@ -1,432 +0,0 @@ -import { mongo } from '@powersync/lib-service-mongodb'; -import { JSONBig } from '@powersync/service-jsonbig'; -import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; -import * as bson from 'bson'; - -import { logger as defaultLogger, Logger } from '@powersync/lib-services-framework'; -import { InternalOpId, storage, utils } from '@powersync/service-core'; -import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; -import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatch.js'; -import { MongoIdSequence } from './MongoIdSequence.js'; -import { VersionedPowerSyncMongo } from './db.js'; -import { - BucketDataDocument, - BucketParameterDocument, - BucketStateDocument, - CurrentBucket, - CurrentDataDocument, - SourceKey -} from './models.js'; - -/** - * Maximum size of operations we write in a single transaction. - * - * It's tricky to find the exact limit, but from experience, over 100MB - * can cause an error: - * > transaction is too large and will not fit in the storage engine cache - * - * Additionally, unbounded size here can balloon our memory usage in some edge - * cases. - * - * When we reach this threshold, we commit the transaction and start a new one. - */ -const MAX_TRANSACTION_BATCH_SIZE = 30_000_000; - -/** - * Limit number of documents to write in a single transaction. - * - * This has an effect on error message size in some cases. - */ -const MAX_TRANSACTION_DOC_COUNT = 2_000; - -/** - * Keeps track of bulkwrite operations within a transaction. - * - * There may be multiple of these batches per transaction, but it may not span - * multiple transactions. - */ -export class PersistedBatch { - logger: Logger; - bucketData: mongo.AnyBulkWriteOperation[] = []; - bucketParameters: mongo.AnyBulkWriteOperation[] = []; - currentData: mongo.AnyBulkWriteOperation[] = []; - bucketStates: Map = new Map(); - - /** - * For debug logging only. - */ - debugLastOpId: InternalOpId | null = null; - - /** - * Very rough estimate of transaction size. - */ - currentSize = 0; - - constructor( - private db: VersionedPowerSyncMongo, - private group_id: number, - writtenSize: number, - options?: { logger?: Logger } - ) { - this.currentSize = writtenSize; - this.logger = options?.logger ?? defaultLogger; - } - - private incrementBucket(bucket: string, op_id: InternalOpId, bytes: number) { - let existingState = this.bucketStates.get(bucket); - if (existingState) { - existingState.lastOp = op_id; - existingState.incrementCount += 1; - existingState.incrementBytes += bytes; - } else { - this.bucketStates.set(bucket, { - lastOp: op_id, - incrementCount: 1, - incrementBytes: bytes - }); - } - } - - saveBucketData(options: { - op_seq: MongoIdSequence; - sourceKey: storage.ReplicaId; - table: storage.SourceTable; - evaluated: EvaluatedRow[]; - before_buckets: CurrentBucket[]; - }) { - const remaining_buckets = new Map(); - for (let b of options.before_buckets) { - const key = currentBucketKey(b); - remaining_buckets.set(key, b); - } - - const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey))); - - for (const k of options.evaluated) { - const key = currentBucketKey(k); - - // INSERT - const recordData = JSONBig.stringify(k.data); - const checksum = utils.hashData(k.table, k.id, recordData); - if (recordData.length > MAX_ROW_SIZE) { - // In many cases, the raw data size would have been too large already. But there are cases where - // the BSON size is small enough, but the JSON size is too large. - // In these cases, we can't store the data, so we skip it, or generate a REMOVE operation if the row - // was synced previously. - this.logger.error(`Row ${key} too large: ${recordData.length} bytes. Removing.`); - continue; - } - - remaining_buckets.delete(key); - const byteEstimate = recordData.length + 200; - this.currentSize += byteEstimate; - - const op_id = options.op_seq.next(); - this.debugLastOpId = op_id; - - this.bucketData.push({ - insertOne: { - document: { - _id: { - g: this.group_id, - b: k.bucket, - o: op_id - }, - op: 'PUT', - source_table: mongoTableId(options.table.id), - source_key: options.sourceKey, - table: k.table, - row_id: k.id, - checksum: BigInt(checksum), - data: recordData - } - } - }); - this.incrementBucket(k.bucket, op_id, byteEstimate); - } - - for (let bd of remaining_buckets.values()) { - // REMOVE - - const op_id = options.op_seq.next(); - this.debugLastOpId = op_id; - - this.bucketData.push({ - insertOne: { - document: { - _id: { - g: this.group_id, - b: bd.bucket, - o: op_id - }, - op: 'REMOVE', - source_table: mongoTableId(options.table.id), - source_key: options.sourceKey, - table: bd.table, - row_id: bd.id, - checksum: dchecksum, - data: null - } - } - }); - this.currentSize += 200; - this.incrementBucket(bd.bucket, op_id, 200); - } - } - - saveParameterData(data: { - op_seq: MongoIdSequence; - sourceKey: storage.ReplicaId; - sourceTable: storage.SourceTable; - evaluated: EvaluatedParameters[]; - existing_lookups: bson.Binary[]; - }) { - // This is similar to saving bucket data. - // A key difference is that we don't need to keep the history intact. - // We do need to keep track of recent history though - enough that we can get consistent data for any specific checkpoint. - // Instead of storing per bucket id, we store per "lookup". - // A key difference is that we don't need to store or keep track of anything per-bucket - the entire record is - // either persisted or removed. - // We also don't need to keep history intact. - const { sourceTable, sourceKey, evaluated } = data; - - const remaining_lookups = new Map(); - for (let l of data.existing_lookups) { - remaining_lookups.set(l.toString('base64'), l); - } - - // 1. Insert new entries - for (let result of evaluated) { - const binLookup = storage.serializeLookup(result.lookup); - const hex = binLookup.toString('base64'); - remaining_lookups.delete(hex); - - const op_id = data.op_seq.next(); - this.debugLastOpId = op_id; - this.bucketParameters.push({ - insertOne: { - document: { - _id: op_id, - key: { - g: this.group_id, - t: mongoTableId(sourceTable.id), - k: sourceKey - }, - lookup: binLookup, - bucket_parameters: result.bucketParameters - } - } - }); - - this.currentSize += 200; - } - - // 2. "REMOVE" entries for any lookup not touched. - for (let lookup of remaining_lookups.values()) { - const op_id = data.op_seq.next(); - this.debugLastOpId = op_id; - this.bucketParameters.push({ - insertOne: { - document: { - _id: op_id, - key: { - g: this.group_id, - t: mongoTableId(sourceTable.id), - k: sourceKey - }, - lookup: lookup, - bucket_parameters: [] - } - } - }); - - this.currentSize += 200; - } - } - - hardDeleteCurrentData(id: SourceKey) { - const op: mongo.AnyBulkWriteOperation = { - deleteOne: { - filter: { _id: id } - } - }; - this.currentData.push(op); - this.currentSize += 50; - } - - /** - * Mark a current_data document as soft deleted, to delete on the next commit. - * - * If softDeleteCurrentData is not enabled, this falls back to a hard delete. - */ - softDeleteCurrentData(id: SourceKey, checkpointGreaterThan: bigint) { - if (!this.db.storageConfig.softDeleteCurrentData) { - this.hardDeleteCurrentData(id); - return; - } - const op: mongo.AnyBulkWriteOperation = { - updateOne: { - filter: { _id: id }, - update: { - $set: { - data: EMPTY_DATA, - buckets: [], - lookups: [], - pending_delete: checkpointGreaterThan - } - }, - upsert: true - } - }; - this.currentData.push(op); - this.currentSize += 50; - } - - upsertCurrentData(id: SourceKey, values: Partial) { - const op: mongo.AnyBulkWriteOperation = { - updateOne: { - filter: { _id: id }, - update: { - $set: values, - $unset: { pending_delete: 1 } - }, - upsert: true - } - }; - this.currentData.push(op); - this.currentSize += (values.data?.length() ?? 0) + 100; - } - - shouldFlushTransaction() { - return ( - this.currentSize >= MAX_TRANSACTION_BATCH_SIZE || - this.bucketData.length >= MAX_TRANSACTION_DOC_COUNT || - this.currentData.length >= MAX_TRANSACTION_DOC_COUNT || - this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT - ); - } - - async flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) { - const db = this.db; - const startAt = performance.now(); - let flushedSomething = false; - if (this.bucketData.length > 0) { - flushedSomething = true; - await db.bucket_data.bulkWrite(this.bucketData, { - session, - // inserts only - order doesn't matter - ordered: false - }); - } - if (this.bucketParameters.length > 0) { - flushedSomething = true; - await db.bucket_parameters.bulkWrite(this.bucketParameters, { - session, - // inserts only - order doesn't matter - ordered: false - }); - } - if (this.currentData.length > 0) { - flushedSomething = true; - await db.common_current_data.bulkWrite(this.currentData, { - session, - // may update and delete data within the same batch - order matters - ordered: true - }); - } - - if (this.bucketStates.size > 0) { - flushedSomething = true; - await db.bucket_state.bulkWrite(this.getBucketStateUpdates(), { - session, - // Per-bucket operation - order doesn't matter - ordered: false - }); - } - - if (flushedSomething) { - const duration = Math.round(performance.now() - startAt); - if (options?.oldestUncommittedChange != null) { - const replicationLag = Math.round((Date.now() - options.oldestUncommittedChange.getTime()) / 1000); - - this.logger.info( - `Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${ - this.currentData.length - } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`, - { - flushed: { - duration: duration, - size: this.currentSize, - bucket_data_count: this.bucketData.length, - parameter_data_count: this.bucketParameters.length, - current_data_count: this.currentData.length, - replication_lag_seconds: replicationLag - } - } - ); - } else { - this.logger.info( - `Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${ - this.currentData.length - } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`, - { - flushed: { - duration: duration, - size: this.currentSize, - bucket_data_count: this.bucketData.length, - parameter_data_count: this.bucketParameters.length, - current_data_count: this.currentData.length - } - } - ); - } - } - - const stats = { - bucketDataCount: this.bucketData.length, - parameterDataCount: this.bucketParameters.length, - currentDataCount: this.currentData.length, - flushedAny: flushedSomething - }; - - this.bucketData = []; - this.bucketParameters = []; - this.currentData = []; - this.bucketStates.clear(); - this.currentSize = 0; - this.debugLastOpId = null; - - return stats; - } - - private getBucketStateUpdates(): mongo.AnyBulkWriteOperation[] { - return Array.from(this.bucketStates.entries()).map(([bucket, state]) => { - return { - updateOne: { - filter: { - _id: { - g: this.group_id, - b: bucket - } - }, - update: { - $set: { - last_op: state.lastOp - }, - $inc: { - 'estimate_since_compact.count': state.incrementCount, - 'estimate_since_compact.bytes': state.incrementBytes - } - }, - upsert: true - } - } satisfies mongo.AnyBulkWriteOperation; - }); - } -} - -interface BucketStateUpdate { - lastOp: InternalOpId; - incrementCount: number; - incrementBytes: number; -} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/BucketDataDoc.ts b/modules/module-mongodb-storage/src/storage/implementation/common/BucketDataDoc.ts new file mode 100644 index 000000000..93e54f6d9 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/BucketDataDoc.ts @@ -0,0 +1,37 @@ +import { InternalOpId } from '@powersync/service-core'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import { BucketDataProperties } from '../models.js'; + +/** + * Full context identifying a bucket. + */ +export interface BucketKey { + /** + * Also referred to as g / group_id. + */ + replicationStreamId: number; + /** + * Bucket definition id, '0' for storage V1. + */ + definitionId: BucketDefinitionId; + /** + * Bucket name. + */ + bucket: string; +} + +/** + * In-memory bucket data document. + * + * This is converted to/from BucketDataDocumentV1 / BucketDataDocumentV3 for storage. + */ +export interface BucketDataDoc extends BucketDataProperties { + /** + * Identifies the bucket for this document. + */ + bucketKey: BucketKey; + /** + * op_id + */ + o: InternalOpId; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts new file mode 100644 index 000000000..ab058b09a --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts @@ -0,0 +1,15 @@ +import { InternalOpId } from '@powersync/service-core'; +import * as bson from 'bson'; +import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; + +export interface MongoSyncBucketStorageContext { + db: TDb; + group_id: number; + mapping: BucketDefinitionMapping; +} + +export interface MongoSyncBucketStorageCheckpoint { + checkpoint: InternalOpId; + snapshotTime: bson.Timestamp; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts new file mode 100644 index 000000000..b7da3c237 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -0,0 +1,364 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { BucketDataSource, EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; + +import { logger as defaultLogger, Logger } from '@powersync/lib-services-framework'; +import { InternalOpId, storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js'; +import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; +import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js'; +import { MongoIdSequence } from '../MongoIdSequence.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; +import { TaggedBucketParameterDocument } from '../models.js'; +import { BucketDataDoc, BucketKey } from './BucketDataDoc.js'; +import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js'; + +/** + * Maximum size of operations we write in a single transaction. + * + * It's tricky to find the exact limit, but from experience, over 100MB + * can cause an error: + * > transaction is too large and will not fit in the storage engine cache + * + * Additionally, unbounded size here can balloon our memory usage in some edge + * cases. + * + * When we reach this threshold, we commit the transaction and start a new one. + */ +const MAX_TRANSACTION_BATCH_SIZE = 30_000_000; + +/** + * Limit number of documents to write in a single transaction. + * + * This has an effect on error message size in some cases. + */ +const MAX_TRANSACTION_DOC_COUNT = 2_000; + +export interface SaveBucketDataOptions { + op_seq: MongoIdSequence; + sourceKey: storage.ReplicaId; + table: storage.SourceTable; + evaluated: EvaluatedRow[]; + before_buckets: SourceRecordBucketState[]; +} + +export interface SaveParameterDataOptions { + op_seq: MongoIdSequence; + sourceKey: storage.ReplicaId; + sourceTable: storage.SourceTable; + evaluated: EvaluatedParameters[]; + existing_lookups: SourceRecordLookupState[]; +} + +export interface UpsertCurrentDataOptions { + sourceTableId: bson.ObjectId; + replicaId: storage.ReplicaId; + data: bson.Binary | null; + buckets: SourceRecordBucketState[]; + lookups: SourceRecordLookupState[]; +} + +export interface PersistedBatchOptions { + logger?: Logger; +} + +/** + * Keeps track of bulkwrite operations within a transaction. + * + * There may be multiple of these batches per transaction, but it may not span + * multiple transactions. + */ +export abstract class PersistedBatch { + logger: Logger; + bucketData: BucketDataDoc[] = []; + bucketParameters: TaggedBucketParameterDocument[] = []; + bucketStates: Map = new Map(); + + /** + * For debug logging only. + */ + debugLastOpId: InternalOpId | null = null; + + /** + * Very rough estimate of transaction size. + */ + currentSize = 0; + + constructor( + protected readonly db: VersionedPowerSyncMongo, + protected readonly group_id: number, + protected readonly mapping: BucketDefinitionMapping, + writtenSize: number, + options?: PersistedBatchOptions + ) { + this.currentSize = writtenSize; + this.logger = options?.logger ?? defaultLogger; + } + + saveBucketData(options: SaveBucketDataOptions) { + const remaining_buckets = new Map(); + for (let bucket of options.before_buckets) { + const mapped: SourceRecordBucketState = { + bucket: bucket.bucket, + definitionId: this.checkDefinitionId(bucket.definitionId), + id: bucket.id, + table: bucket.table + }; + remaining_buckets.set(currentBucketKey(mapped), mapped); + } + + const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey))); + + for (const evaluated of options.evaluated) { + const definitionId = this.getBucketDefinitionId(evaluated.source); + const key = currentBucketKey({ + definitionId: definitionId, + bucket: evaluated.bucket, + table: evaluated.table, + id: evaluated.id + }); + + const recordData = JSONBig.stringify(evaluated.data); + const checksum = utils.hashData(evaluated.table, evaluated.id, recordData); + if (recordData.length > MAX_ROW_SIZE) { + this.logger.error(`Row ${key} too large: ${recordData.length} bytes. Removing.`); + continue; + } + + remaining_buckets.delete(key); + const byteEstimate = recordData.length + 200; + this.currentSize += byteEstimate; + + const op_id = options.op_seq.next(); + this.debugLastOpId = op_id; + + this.addBucketDataPut({ + bucketKey: { + bucket: evaluated.bucket, + definitionId: definitionId, + replicationStreamId: this.group_id + }, + op_id, + bucket: evaluated.bucket, + sourceTableId: options.table.id, + sourceKey: options.sourceKey, + table: evaluated.table, + rowId: evaluated.id, + checksum: BigInt(checksum), + data: recordData + }); + this.incrementBucket(definitionId, evaluated.bucket, op_id, byteEstimate); + } + + for (let bucket of remaining_buckets.values()) { + const definitionId = bucket.definitionId!; + const op_id = options.op_seq.next(); + this.debugLastOpId = op_id; + + this.addBucketDataRemove({ + bucketKey: { + replicationStreamId: this.group_id, + definitionId, + bucket: bucket.bucket + }, + op_id, + sourceTableId: options.table.id, + sourceKey: options.sourceKey, + table: bucket.table, + rowId: bucket.id, + checksum: dchecksum + }); + this.currentSize += 200; + this.incrementBucket(definitionId, bucket.bucket, op_id, 200); + } + } + + abstract saveParameterData(data: SaveParameterDataOptions): void; + + abstract hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): void; + + abstract softDeleteCurrentData( + sourceTableId: bson.ObjectId, + replicaId: storage.ReplicaId, + checkpointGreaterThan: bigint + ): void; + + abstract upsertCurrentData(values: UpsertCurrentDataOptions): void; + + protected abstract get currentDataCount(): number; + + protected abstract flushBucketData(session: mongo.ClientSession): Promise; + + protected abstract flushBucketParameters(session: mongo.ClientSession): Promise; + + protected abstract flushCurrentData(session: mongo.ClientSession): Promise; + + protected abstract flushBucketStates(session: mongo.ClientSession): Promise; + + protected abstract resetCurrentData(): void; + + protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId; + protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId; + + protected get bucketDataCount(): number { + return this.bucketData.length; + } + + protected incrementBucket(definitionId: BucketDefinitionId, bucket: string, op_id: InternalOpId, bytes: number) { + const key = `${definitionId ?? ''}:${bucket}`; + let existingState = this.bucketStates.get(key); + if (existingState) { + existingState.lastOp = op_id; + existingState.incrementCount += 1; + existingState.incrementBytes += bytes; + } else { + this.bucketStates.set(key, { + definitionId, + bucket, + lastOp: op_id, + incrementCount: 1, + incrementBytes: bytes + }); + } + } + + protected addBucketDataPut(options: { + op_id: InternalOpId; + bucketKey: BucketKey; + bucket: string; + sourceTableId: storage.SourceTable['id']; + sourceKey: storage.ReplicaId; + table: string; + rowId: string; + checksum: bigint; + data: string; + }) { + this.bucketData.push({ + bucketKey: options.bucketKey, + o: options.op_id, + op: 'PUT', + source_table: mongoTableId(options.sourceTableId), + source_key: options.sourceKey, + table: options.table, + row_id: options.rowId, + checksum: options.checksum, + data: options.data + }); + } + + protected addBucketDataRemove(options: { + op_id: InternalOpId; + bucketKey: BucketKey; + sourceTableId: storage.SourceTable['id']; + sourceKey: storage.ReplicaId; + table: string; + rowId: string; + checksum: bigint; + }) { + this.bucketData.push({ + bucketKey: options.bucketKey, + o: options.op_id, + op: 'REMOVE', + source_table: mongoTableId(options.sourceTableId), + source_key: options.sourceKey, + table: options.table, + row_id: options.rowId, + checksum: options.checksum, + data: null + }); + } + + shouldFlushTransaction() { + return ( + this.currentSize >= MAX_TRANSACTION_BATCH_SIZE || + this.bucketDataCount >= MAX_TRANSACTION_DOC_COUNT || + this.currentDataCount >= MAX_TRANSACTION_DOC_COUNT || + this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT + ); + } + + async flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) { + const startAt = performance.now(); + let flushedSomething = false; + if (this.bucketDataCount > 0) { + flushedSomething = true; + await this.flushBucketData(session); + } + if (this.bucketParameters.length > 0) { + flushedSomething = true; + await this.flushBucketParameters(session); + } + if (this.currentDataCount > 0) { + flushedSomething = true; + await this.flushCurrentData(session); + } + + if (this.bucketStates.size > 0) { + flushedSomething = true; + await this.flushBucketStates(session); + } + + if (flushedSomething) { + const duration = Math.round(performance.now() - startAt); + if (options?.oldestUncommittedChange != null) { + const replicationLag = Math.round((Date.now() - options.oldestUncommittedChange.getTime()) / 1000); + + this.logger.info( + `Flushed ${this.bucketDataCount} + ${this.bucketParameters.length} + ${ + this.currentDataCount + } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`, + { + flushed: { + duration: duration, + size: this.currentSize, + bucket_data_count: this.bucketDataCount, + parameter_data_count: this.bucketParameters.length, + current_data_count: this.currentDataCount, + replication_lag_seconds: replicationLag + } + } + ); + } else { + this.logger.info( + `Flushed ${this.bucketDataCount} + ${this.bucketParameters.length} + ${ + this.currentDataCount + } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`, + { + flushed: { + duration: duration, + size: this.currentSize, + bucket_data_count: this.bucketDataCount, + parameter_data_count: this.bucketParameters.length, + current_data_count: this.currentDataCount + } + } + ); + } + } + + const stats = { + bucketDataCount: this.bucketDataCount, + parameterDataCount: this.bucketParameters.length, + currentDataCount: this.currentDataCount, + flushedAny: flushedSomething + }; + + this.bucketData = []; + this.bucketParameters = []; + this.resetCurrentData(); + this.bucketStates.clear(); + this.currentSize = 0; + this.debugLastOpId = null; + + return stats; + } +} + +export interface BucketStateUpdate { + definitionId: BucketDefinitionId | null; + bucket: string; + lastOp: InternalOpId; + incrementCount: number; + incrementBytes: number; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/SingleBucketStore.ts b/modules/module-mongodb-storage/src/storage/implementation/common/SingleBucketStore.ts new file mode 100644 index 000000000..af04ad23c --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/SingleBucketStore.ts @@ -0,0 +1,63 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { InternalOpId } from '@powersync/service-core'; +import { BucketDataProperties } from '../models.js'; +import { BucketDataDoc, BucketKey } from './BucketDataDoc.js'; + +const GENERIC_ID = Symbol('BucketDataDocumentGenericId'); +export type BucketDataDocumentGenericId = { + b: string; + o: InternalOpId; + // Hack to ensure this can't be constructed directly + [GENERIC_ID]: true; +}; + +/** + * This document is never actually constructed - we use it as a "virtual" type. + * + * The actual implementations are BucketDataDocumentV1 or BucketDataDocumentV3. + * They don't fully satisfy this interface, but this works to share common implementations. + * + * The idea is that we can have a common implementation between V1 & V3, using this type, + * and operate on MongoDB collections. + * + * This interface serves two primary purposes: + * 1. Captures properties that exist on both V1 and V3 storage models. + * 2. Gives a common reference when querying or modifying collections. + * + * Generics would've been ideal, but they don't play well with MongoDB collections. + */ +export interface BucketDataDocumentGeneric extends BucketDataProperties { + _id: BucketDataDocumentGenericId; +} + +/** + * Represent read/write access for a single bucket. + * + * This does not implement the actual collection operations, but supports the required conversions + * between in-memory BucketDataDoc and the specific storage formats. + */ +export interface SingleBucketStore { + readonly key: BucketKey; + + readonly collection: mongo.Collection; + docId(o: InternalOpId): BucketDataDocumentGenericId; + readonly minId: BucketDataDocumentGenericId; + readonly maxId: BucketDataDocumentGenericId; + + /** + * Convert in-memory document -> persisted document. + */ + toPersistedDocument(source: Omit): BucketDataDocumentGeneric; + + /** + * Convert persisted document -> in-memory document. + */ + fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc; + + /** + * Convert partial persisted document -> partial in-memory document. + */ + fromPartialPersistedDocument( + doc: Pick + ): Pick; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/SourceRecordStore.ts b/modules/module-mongodb-storage/src/storage/implementation/common/SourceRecordStore.ts new file mode 100644 index 000000000..0984a3632 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/SourceRecordStore.ts @@ -0,0 +1,49 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { Logger } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; +import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; + +export interface SourceRecordLookupEntry { + sourceTableId: bson.ObjectId; + replicaId: storage.ReplicaId; +} + +export interface SourceRecordBucketState { + definitionId: BucketDefinitionId | null; + bucket: string; + table: string; + id: string; +} + +export interface SourceRecordLookupState { + indexId: ParameterIndexId | null; + lookup: bson.Binary; +} + +export interface LoadedSourceRecord { + sourceTableId: bson.ObjectId; + replicaId: storage.ReplicaId; + data: bson.Binary | null; + buckets: SourceRecordBucketState[]; + lookups: SourceRecordLookupState[]; + cacheKey: string; +} + +export interface SourceRecordStore { + mapEvaluatedBuckets(evaluated: EvaluatedRow[]): SourceRecordBucketState[]; + mapParameterLookups(paramEvaluated: EvaluatedParameters[]): SourceRecordLookupState[]; + loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise>; + loadDocuments( + session: mongo.ClientSession, + entries: SourceRecordLookupEntry[], + idsOnly: boolean + ): Promise>; + loadTruncateBatch( + session: mongo.ClientSession, + sourceTableId: bson.ObjectId, + limit: number + ): Promise; + postCommitCleanup(lastCheckpoint: bigint, logger: Logger): Promise; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts new file mode 100644 index 000000000..1e65d2e95 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts @@ -0,0 +1,80 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { DO_NOT_LOG } from '@powersync/lib-services-framework'; +import { PowerSyncMongo } from '../db.js'; +import { CommonSourceTableDocument, StorageConfig } from '../models.js'; + +export abstract class BaseVersionedPowerSyncMongo { + readonly client: mongo.MongoClient; + readonly db: mongo.Db; + readonly storageConfig: StorageConfig; + [DO_NOT_LOG] = true; + + constructor( + protected readonly upstream: PowerSyncMongo, + storageConfig: StorageConfig + ) { + this.client = upstream.client; + this.db = upstream.db; + this.storageConfig = storageConfig; + } + + get bucket_data() { + return this.upstream.bucket_data; + } + + get op_id_sequence() { + return this.upstream.op_id_sequence; + } + + get sync_rules() { + return this.upstream.sync_rules; + } + + get custom_write_checkpoints() { + return this.upstream.custom_write_checkpoints; + } + + get write_checkpoints() { + return this.upstream.write_checkpoints; + } + + get instance() { + return this.upstream.instance; + } + + get locks() { + return this.upstream.locks; + } + + get checkpoint_events() { + return this.upstream.checkpoint_events; + } + + get connection_report_events() { + return this.upstream.connection_report_events; + } + + notifyCheckpoint() { + return this.upstream.notifyCheckpoint(); + } + + protected sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) { + return this.upstream.sourceRecordsCollectionName(replicationStreamId, sourceTableId); + } + + protected sourceTableCollectionName(replicationStreamId: number) { + return this.upstream.sourceTableCollectionName(replicationStreamId); + } + + protected async listCollectionsByPrefix(prefix: string): Promise[]> { + const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => this.db.collection(collection.name)); + } + + abstract commonSourceTables(replicationStreamId: number): mongo.Collection; + + abstract initializeStreamStorage(replicationStreamId: number): Promise; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts new file mode 100644 index 000000000..569a5eb58 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts @@ -0,0 +1,25 @@ +import { storage } from '@powersync/service-core'; +import { MongoBucketStorage } from '../MongoBucketStorage.js'; +import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js'; +import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js'; +import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js'; +import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js'; + +export { MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js'; + +export type { MongoSyncBucketStorage }; + +export function createMongoSyncBucketStorage( + factory: MongoBucketStorage, + group_id: number, + sync_rules: MongoPersistedSyncRulesContent, + slot_name: string, + writeCheckpointMode: storage.WriteCheckpointMode | undefined, + options: MongoSyncBucketStorageOptions +): MongoSyncBucketStorage { + if (sync_rules.getStorageConfig().incrementalReprocessing) { + return new MongoSyncBucketStorageV3(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); + } + + return new MongoSyncBucketStorageV1(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index de37a8a82..a3b54ca75 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -2,16 +2,13 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { POWERSYNC_VERSION, storage } from '@powersync/service-core'; -import { DO_NOT_LOG, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { DO_NOT_LOG } from '@powersync/lib-services-framework'; import { MongoStorageConfig } from '../../types/types.js'; +import { BaseVersionedPowerSyncMongo } from './common/VersionedPowerSyncMongoBase.js'; import { - BucketDataDocument, - BucketParameterDocument, - BucketStateDocument, CheckpointEventDocument, ClientConnectionDocument, - CurrentDataDocument, - CurrentDataDocumentV3, + CommonSourceTableDocument, CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, @@ -20,6 +17,15 @@ import { SyncRuleDocument, WriteCheckpointDocument } from './models.js'; +import { + BucketDataDocumentV1, + BucketParameterDocument, + BucketStateDocumentV1, + CurrentDataDocument +} from './v1/models.js'; +import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js'; +import { BucketDataDocumentV3 } from './v3/models.js'; +import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js'; export interface PowerSyncMongoOptions { /** @@ -32,8 +38,7 @@ export class PowerSyncMongo { [DO_NOT_LOG] = true; readonly current_data: mongo.Collection; - readonly v3_current_data: mongo.Collection; - readonly bucket_data: mongo.Collection; + readonly bucket_data: mongo.Collection; readonly bucket_parameters: mongo.Collection; readonly op_id_sequence: mongo.Collection; readonly sync_rules: mongo.Collection; @@ -42,7 +47,7 @@ export class PowerSyncMongo { readonly write_checkpoints: mongo.Collection; readonly instance: mongo.Collection; readonly locks: mongo.Collection; - readonly bucket_state: mongo.Collection; + readonly bucket_state: mongo.Collection; readonly checkpoint_events: mongo.Collection; readonly connection_report_events: mongo.Collection; @@ -58,7 +63,6 @@ export class PowerSyncMongo { this.db = db; this.current_data = db.collection('current_data'); - this.v3_current_data = db.collection('v3_current_data'); this.bucket_data = db.collection('bucket_data'); this.bucket_parameters = db.collection('bucket_parameters'); this.op_id_sequence = db.collection('op_id_sequence'); @@ -73,8 +77,80 @@ export class PowerSyncMongo { this.connection_report_events = this.db.collection('connection_report_events'); } - versioned(storageConfig: StorageConfig) { - return new VersionedPowerSyncMongo(this, storageConfig); + versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo { + if (storageConfig.incrementalReprocessing) { + return new VersionedPowerSyncMongoV3(this, storageConfig); + } + + return new VersionedPowerSyncMongoV1(this, storageConfig); + } + + /** + * Not safe for user-provided prefix - only for hardcoded values. + */ + async listBucketDataCollectionsV3(groupId?: number): Promise[]> { + const prefix = groupId == null ? 'bucket_data_' : `bucket_data_${groupId}_`; + const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => this.db.collection(collection.name)); + } + + /** + * Not safe for user-provided prefix - only for hardcoded values. + */ + private async collectionsByPrefix(prefix: string): Promise[]> { + const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => this.db.collection(collection.name)); + } + + /** + * List all parameter index collections across all replication streams. + * + * Primarily used to clear the db. + */ + async listAllParameterIndexCollectionsV3(): Promise[]> { + return this.collectionsByPrefix(`parameter_index_`); + } + + /** + * List all parameter index collections across all replication streams. + * + * Primarily used to clear the db. + */ + async listAllSourceRecordCollectionsV3(): Promise[]> { + return this.collectionsByPrefix(`source_records_`); + } + + async listAllBucketStateCollectionsV3(): Promise[]> { + return this.collectionsByPrefix(`bucket_state_`); + } + + sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) { + return `source_records_${replicationStreamId}_${sourceTableId.toHexString()}`; + } + + sourceTableCollectionName(replicationStreamId: number) { + return `source_table_${replicationStreamId}`; + } + + async listSourceTableCollections( + replicationStreamId?: number + ): Promise[]> { + const filter = + replicationStreamId == null + ? { name: new RegExp('^source_table_') } + : { name: this.sourceTableCollectionName(replicationStreamId) }; + const prefix = replicationStreamId == null ? 'source_table_' : this.sourceTableCollectionName(replicationStreamId); + const collections = await this.db.listCollections(filter, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => this.db.collection(collection.name)); } /** @@ -82,11 +158,25 @@ export class PowerSyncMongo { */ async clear() { await this.current_data.deleteMany({}); - await this.v3_current_data.deleteMany({}); + for (const collection of await this.listAllSourceRecordCollectionsV3()) { + await collection.drop(); + } await this.bucket_data.deleteMany({}); + for (const collection of await this.listBucketDataCollectionsV3()) { + await collection.drop(); + } await this.bucket_parameters.deleteMany({}); + for (const collection of await this.listAllParameterIndexCollectionsV3()) { + await collection.drop(); + } + for (const collection of await this.listAllBucketStateCollectionsV3()) { + await collection.drop(); + } await this.op_id_sequence.deleteMany({}); await this.sync_rules.deleteMany({}); + for (const collection of await this.listSourceTableCollections()) { + await collection.drop(); + } await this.source_tables.deleteMany({}); await this.write_checkpoints.deleteMany({}); await this.instance.deleteOne({}); @@ -183,126 +273,12 @@ export class PowerSyncMongo { { name: 'dirty_count' } ); } - - async initializeStorageVersion(storageConfig: StorageConfig) { - if (storageConfig.softDeleteCurrentData) { - // Initialize the v3_current_data collection, which is used for the new storage version. - // No-op if this already exists - await this.v3_current_data.createIndex( - { - '_id.g': 1, - pending_delete: 1 - }, - { - partialFilterExpression: { pending_delete: { $exists: true } }, - name: 'pending_delete' - } - ); - } - } } /** * This is similar to PowerSyncMongo, but blocks access to certain collections based on the storage version. */ -export class VersionedPowerSyncMongo { - readonly client: mongo.MongoClient; - readonly db: mongo.Db; - [DO_NOT_LOG] = true; - - readonly storageConfig: StorageConfig; - #upstream: PowerSyncMongo; - - constructor(upstream: PowerSyncMongo, storageConfig: StorageConfig) { - this.#upstream = upstream; - this.client = upstream.client; - this.db = upstream.db; - this.storageConfig = storageConfig; - } - - /** - * Uses either `current_data` or `v3_current_data` collection based on the storage version. - * - * Use in places where it does not matter which version is used. - */ - get common_current_data(): mongo.Collection { - if (this.storageConfig.softDeleteCurrentData) { - return this.#upstream.v3_current_data; - } else { - return this.#upstream.current_data; - } - } - - get v1_current_data() { - if (this.storageConfig.softDeleteCurrentData) { - throw new ServiceAssertionError( - 'current_data collection should not be used when softDeleteCurrentData is enabled' - ); - } - return this.#upstream.current_data; - } - - get v3_current_data() { - if (!this.storageConfig.softDeleteCurrentData) { - throw new ServiceAssertionError( - 'v3_current_data collection should not be used when softDeleteCurrentData is disabled' - ); - } - return this.#upstream.v3_current_data; - } - - get bucket_data() { - return this.#upstream.bucket_data; - } - - get bucket_parameters() { - return this.#upstream.bucket_parameters; - } - - get op_id_sequence() { - return this.#upstream.op_id_sequence; - } - - get sync_rules() { - return this.#upstream.sync_rules; - } - - get source_tables() { - return this.#upstream.source_tables; - } - - get custom_write_checkpoints() { - return this.#upstream.custom_write_checkpoints; - } - - get write_checkpoints() { - return this.#upstream.write_checkpoints; - } - - get instance() { - return this.#upstream.instance; - } - - get locks() { - return this.#upstream.locks; - } - - get bucket_state() { - return this.#upstream.bucket_state; - } - - get checkpoint_events() { - return this.#upstream.checkpoint_events; - } - - get connection_report_events() { - return this.#upstream.connection_report_events; - } - - notifyCheckpoint() { - return this.#upstream.notifyCheckpoint(); - } -} +export type VersionedPowerSyncMongo = BaseVersionedPowerSyncMongo; export function createPowerSyncMongo(config: MongoStorageConfig, options?: lib_mongo.MongoConnectionOptions) { return new PowerSyncMongo( diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 0e76d7d8e..6c09fc692 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -3,6 +3,9 @@ import { InternalOpId, SerializedSyncPlan, storage } from '@powersync/service-co import { SqliteJsonValue } from '@powersync/service-sync-rules'; import { event_types } from '@powersync/service-types'; import * as bson from 'bson'; +import { ParameterIndexId } from './BucketDefinitionMapping.js'; +import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js'; +import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from './v3/models.js'; /** * Replica id uniquely identifying a row on the source database. @@ -22,50 +25,53 @@ export interface SourceKey { k: ReplicaId; } +export interface SourceTableKey { + /** source table id */ + t: bson.ObjectId; + /** source key */ + k: ReplicaId; +} + export interface BucketDataKey { - /** group_id */ - g: number; /** bucket name */ b: string; /** op_id */ o: bigint; } -export interface CurrentDataDocument { - _id: SourceKey; - data: bson.Binary; - buckets: CurrentBucket[]; - lookups: bson.Binary[]; -} - -export interface CurrentDataDocumentV3 { - _id: SourceKey; - data: bson.Binary; - buckets: CurrentBucket[]; - lookups: bson.Binary[]; - /** - * If set, this can be deleted, once there is a consistent checkpoint >= pending_delete. - * - * This must only be set if buckets = [], lookups = []. - */ - pending_delete?: bigint; -} - export interface CurrentBucket { bucket: string; table: string; id: string; } -export interface BucketParameterDocument { +export interface BucketParameterDocumentBase { _id: bigint; - key: SourceKey; + key: TKey; lookup: bson.Binary; bucket_parameters: Record[]; } -export interface BucketDataDocument { - _id: BucketDataKey; +export interface TaggedBucketParameterDocument extends BucketParameterDocumentBase { + index: ParameterIndexId; +} + +export function bucketParameterDocumentToTagged( + document: BucketParameterDocumentBase, + index: ParameterIndexId +): TaggedBucketParameterDocument { + return { + ...document, + index + }; +} + +export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR'; + +/** + * Common properties for storage V1, storage V3 and in-memory BucketDataDoc. + */ +export interface BucketDataProperties { op: OpType; source_table?: bson.ObjectId; source_key?: ReplicaId; @@ -76,11 +82,22 @@ export interface BucketDataDocument { target_op?: bigint | null; } -export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR'; +export interface BucketDataDocumentBase extends BucketDataProperties { + _id: { b: string }; +} + +/** + * Internal-only tag used for v1 bucket_data rows before they are converted to the v1 on-disk shape. + */ +export const LEGACY_BUCKET_DATA_DEFINITION_ID = '0'; + +/** + * Internal-only tag used for v1 bucket_parameters rows before they are converted to the v1 on-disk shape. + */ +export const LEGACY_BUCKET_PARAMETER_INDEX_ID = '0'; export interface SourceTableDocument { _id: bson.ObjectId; - group_id: number; connection_id: number; relation_id: number | string | undefined; schema_name: string; @@ -100,20 +117,21 @@ export interface SourceTableDocumentSnapshotStatus { /** * Record the state of each bucket. * - * Right now, this is just used to track when buckets are updated, for efficient incremental sync. - * In the future, this could be used to track operation counts, both for diagnostic purposes, and for - * determining when a compact and/or defragment could be beneficial. + * The primary use case is to track when buckets are updated, for efficient incremental sync. * - * Note: There is currently no migration to populate this collection from existing data - it is only + * The secondary use case is to track operation counts to determine whether or not a bucket should be compacted. + * + * Note: For storage V1, there is no migration to populate this collection from existing data - it is only * populated by new updates. + * + * For storage V3, these will always be present. */ -export interface BucketStateDocument { +export interface BucketStateDocumentBase { _id: { - g: number; b: string; }; /** - * Important: There is an unique index on {'_id.g': 1, last_op: 1}. + * Important: There is an unique index on last_op per logical stream. * That means the last_op must match an actual op in the bucket, and not the commit checkpoint. */ last_op: bigint; @@ -215,6 +233,20 @@ export interface SyncRuleDocument { content: string; serialized_plan?: SerializedSyncPlan | null; + /** + * Required for V3+ storage. + */ + rule_mapping?: { + /** + * Map of uniqueName -> id, unique per replication stream. + */ + definitions: Record; + /** + * Map of (lookupName, queryId) -> id, unique per replication stream. + */ + parameter_indexes: Record; + }; + lock?: { id: string; expires_at: Date; @@ -231,9 +263,14 @@ export interface StorageConfig extends storage.StorageVersionConfig { * a Long before summing. */ longChecksums: boolean; + /** + * Enables v3 MongoDB storage behavior used for incremental reprocessing. + */ + incrementalReprocessing: boolean; } const LONG_CHECKSUMS_STORAGE_VERSION = 2; +const INCREMENTAL_REPROCESSING_STORAGE_VERSION = storage.STORAGE_VERSION_3; export function getMongoStorageConfig(storageVersion: number): StorageConfig { const baseConfig = storage.STORAGE_VERSION_CONFIG[storageVersion]; @@ -241,7 +278,11 @@ export function getMongoStorageConfig(storageVersion: number): StorageConfig { throw new ServiceError(ErrorCode.PSYNC_S1005, `Unsupported storage version ${storageVersion}`); } - return { ...baseConfig, longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION }; + return { + ...baseConfig, + longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION, + incrementalReprocessing: storageVersion >= INCREMENTAL_REPROCESSING_STORAGE_VERSION + }; } export interface CheckpointEventDocument { @@ -286,3 +327,8 @@ export interface InstanceDocument { } export interface ClientConnectionDocument extends event_types.ClientConnection {} + +export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocumentV3['_id']; +export type CommonCurrentBucket = CurrentBucket | CurrentBucketV3; +export type CommonCurrentLookup = bson.Binary | RecordedLookupV3; +export type CommonSourceTableDocument = SourceTableDocumentV1 | SourceTableDocumentV3; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts new file mode 100644 index 000000000..723aa371b --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts @@ -0,0 +1,32 @@ +import { SourceTable } from '@powersync/service-core'; +import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js'; +import { PersistedBatch } from '../common/PersistedBatch.js'; +import { SourceRecordStore } from '../common/SourceRecordStore.js'; +import { PersistedBatchV1 } from './PersistedBatchV1.js'; +import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; + +export class MongoBucketBatchV1 extends MongoBucketBatch { + declare public readonly db: VersionedPowerSyncMongoV1; + + private readonly store: SourceRecordStore; + + constructor(options: MongoBucketBatchOptions) { + super(options); + this.store = new SourceRecordStoreV1(this.db, this.group_id); + } + + protected createPersistedBatch(writtenSize: number): PersistedBatch { + return new PersistedBatchV1(this.db, this.group_id, this.mapping, writtenSize, { + logger: this.logger + }); + } + + protected get sourceRecordStore(): SourceRecordStore { + return this.store; + } + + protected async cleanupDroppedSourceTables(_tables: SourceTable[]) { + // No-op for V1: source records live in a shared collection. + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts new file mode 100644 index 000000000..62636d37c --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts @@ -0,0 +1,75 @@ +import { + bson, + BucketChecksum, + FetchPartialBucketChecksum, + InternalOpId, + PartialChecksumMap +} from '@powersync/service-core'; +import { FetchPartialBucketChecksumByBucket, MongoChecksums } from '../MongoChecksums.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; + +export class MongoChecksumsV1 extends MongoChecksums { + declare protected readonly db: VersionedPowerSyncMongoV1; + + async computePartialChecksumsDirectByBucket( + batch: FetchPartialBucketChecksumByBucket[] + ): Promise { + return this.computePartialChecksumsForCollection(batch, this.db.bucketDataV1, (request) => ({ + _id: { + $gt: { + g: this.group_id, + b: request.bucket, + o: request.start ?? new bson.MinKey() + }, + $lte: { + g: this.group_id, + b: request.bucket, + o: request.end + } + } + })); + } + + protected async fetchPreStates( + batch: FetchPartialBucketChecksum[] + ): Promise> { + const preFilters = batch + .filter((request) => request.start == null) + .map((request) => ({ + _id: { + g: this.group_id, + b: request.bucket + }, + 'compacted_state.op_id': { $exists: true, $lte: request.end } + })); + + const preStates = new Map(); + if (preFilters.length == 0) { + return preStates; + } + + const states = await this.db.bucketStateV1 + .find({ + $or: preFilters + }) + .toArray(); + + for (const state of states) { + const compactedState = state.compacted_state!; + preStates.set(state._id.b, { + opId: compactedState.op_id, + checksum: { + bucket: state._id.b, + checksum: Number(compactedState.checksum), + count: compactedState.count + } + }); + } + + return preStates; + } + + protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { + return this.computePartialChecksumsDirectByBucket(batch); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts new file mode 100644 index 000000000..b19e94c53 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -0,0 +1,93 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import { SingleBucketStore } from '../common/SingleBucketStore.js'; +import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js'; +import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; +import { BucketStateDocumentV1 } from './models.js'; +import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js'; +import { SingleBucketStoreV1 } from './SingleBucketStoreV1.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; + +export class MongoCompactorV1 extends MongoCompactor { + // Override types to the more specific ones + declare protected readonly db: VersionedPowerSyncMongoV1; + declare protected readonly storage: MongoSyncBucketStorageV1; + + public async *dirtyBucketBatches(options: { + minBucketChanges: number; + minChangeRatio: number; + }): AsyncGenerator { + if (options.minBucketChanges <= 0) { + throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + } + // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only scan buckets with changes. + // That works well if there are only a small number of dirty buckets, but it causes repeated rescans while data is + // still changing. We now iterate through all V1 bucket_state rows for the group and filter after projecting. + yield* this.dirtyBucketBatchesForCollection( + this.db.bucketStateV1, + { g: this.group_id, b: new mongo.MinKey() as any }, + { g: this.group_id, b: new mongo.MaxKey() as any }, + options, + () => null + ); + } + + public async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { + if (options.minBucketChanges <= 0) { + throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + } + // Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets + // estimate_since_compact as it progresses. + return this.dirtyBucketBatchForChecksumsForCollection( + this.db.bucketStateV1, + { + '_id.g': this.group_id, + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + }, + () => null + ); + } + + protected async writeBucketStateUpdates(): Promise { + await this.db.bucketStateV1.bulkWrite( + this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], + { ordered: false } + ); + } + + protected async computeChecksumsForBuckets( + buckets: Pick[] + ): Promise { + return this.storage.checksums.computePartialChecksumsDirectByBucket( + buckets.map(({ bucket }) => ({ + bucket, + end: this.maxOpId + })) + ); + } + + protected bucketStateFilter( + bucket: string, + _definitionId: BucketDefinitionId | null + ): mongo.Filter { + return { + _id: { + g: this.group_id, + b: bucket + } + }; + } + + protected async getBucketDataContext( + bucket: string, + _definitionId: BucketDefinitionId | null + ): Promise { + return new SingleBucketStoreV1(this.db, { + replicationStreamId: this.group_id, + definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID, + bucket + }); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts new file mode 100644 index 000000000..4ff5f4cb6 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts @@ -0,0 +1,26 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; + +export class MongoParameterCompactorV1 extends MongoParameterCompactor { + declare protected readonly db: VersionedPowerSyncMongoV1; + + protected async getCollections(): Promise[]> { + return [this.db.parameterIndexV1 as unknown as mongo.Collection]; + } + + protected collectionFilter(): mongo.Document { + return { + 'key.g': this.group_id + }; + } + + protected deleteFilter(doc: mongo.Document): mongo.Document { + return { + 'key.g': doc.key.g as number, + lookup: doc.lookup, + _id: { $lte: doc._id }, + key: doc.key + }; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts new file mode 100644 index 000000000..77fae1869 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -0,0 +1,423 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; +import { + CheckpointChanges, + deserializeParameterLookup, + GetCheckpointChangesOptions, + InternalOpId, + internalToExternalOpId, + ProtocolOpId, + storage, + utils +} from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; +import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; +import { MongoBucketStorage } from '../../MongoBucketStorage.js'; +import { + MongoSyncBucketStorageCheckpoint, + MongoSyncBucketStorageContext +} from '../common/MongoSyncBucketStorageContext.js'; +import { CommonSourceTableDocument, SourceKey } from '../models.js'; +import { MongoBucketBatchOptions } from '../MongoBucketBatch.js'; +import { MongoChecksums } from '../MongoChecksums.js'; +import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; +import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; +import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; +import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; +import { BucketDataDocumentV1, BucketDataKeyV1, BucketStateDocument, loadBucketDataDocumentV1 } from './models.js'; +import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; +import { MongoChecksumsV1 } from './MongoChecksumsV1.js'; +import { MongoCompactorV1 } from './MongoCompactorV1.js'; +import { MongoParameterCompactorV1 } from './MongoParameterCompactorV1.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; + +export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { + // Declare types to be more specific + declare readonly db: VersionedPowerSyncMongoV1; + declare readonly checksums: MongoChecksumsV1; + + constructor( + factory: MongoBucketStorage, + group_id: number, + sync_rules: MongoPersistedSyncRulesContent, + slot_name: string, + writeCheckpointMode: storage.WriteCheckpointMode | undefined, + options: MongoSyncBucketStorageOptions + ) { + super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); + } + + protected async initializeVersionStorage(): Promise {} + + protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch { + return new MongoBucketBatchV1(batchOptions); + } + + protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums { + return new MongoChecksumsV1(this.db, this.group_id, { + ...options.checksumOptions, + storageConfig: options?.storageConfig, + mapping: this.sync_rules.mapping + }); + } + + createMongoCompactor(options: MongoCompactOptions): MongoCompactor { + return new MongoCompactorV1(this, this.db, options); + } + + protected createMongoParameterCompactor( + checkpoint: InternalOpId, + options: storage.CompactOptions + ): MongoParameterCompactor { + return new MongoParameterCompactorV1(this.db, this.group_id, checkpoint, options); + } + + protected sourceTableBaseId(): Partial { + return { group_id: this.group_id }; + } + + protected augmentCreatedSourceTableDocument( + _createDoc: CommonSourceTableDocument, + _options: storage.ResolveTableOptions, + _candidateSourceTable: storage.SourceTable + ): void {} + + protected async initializeResolvedSourceRecords(_sourceTableId: bson.ObjectId): Promise {} + + protected override get versionContext(): MongoSyncBucketStorageContext { + return { + db: this.db, + group_id: this.group_id, + mapping: this.mapping + }; + } + + protected getParameterSetsImpl( + checkpoint: MongoSyncBucketStorageCheckpoint, + lookups: ScopedParameterLookup[] + ): Promise { + return getParameterSetsV1(this.versionContext, checkpoint, lookups); + } + + protected getBucketDataBatchImpl( + checkpoint: utils.InternalOpId, + dataBuckets: storage.BucketDataRequest[], + options?: storage.BucketDataBatchOptions + ): AsyncIterable { + return getBucketDataBatchV1(this.versionContext, checkpoint, dataBuckets, options); + } + + protected async clearBucketData(signal?: AbortSignal): Promise { + await this.clearDeleteMany( + 'bucket data', + () => + this.db.bucket_data.deleteMany( + { + _id: idPrefixFilter({ g: this.group_id }, ['b', 'o']) + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal + ); + } + + protected async clearParameterIndexes(signal?: AbortSignal): Promise { + await this.clearDeleteMany( + 'parameter index', + () => + this.db.parameterIndexV1.deleteMany( + { + 'key.g': this.group_id + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal + ); + } + + protected async clearSourceRecords(signal?: AbortSignal): Promise { + await this.clearDeleteMany( + 'source records', + () => + this.db.sourceRecordsV1.deleteMany( + { + _id: idPrefixFilter({ g: this.group_id }, ['t', 'k']) + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal + ); + } + + protected async clearBucketState(signal?: AbortSignal): Promise { + await this.clearDeleteMany( + 'bucket state', + () => + this.db.bucketStateV1.deleteMany( + { + _id: idPrefixFilter({ g: this.group_id }, ['b']) + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal + ); + } + + protected async clearSourceTables(signal?: AbortSignal): Promise { + await this.clearDeleteMany( + 'source tables', + () => + this.db.commonSourceTables(this.group_id).deleteMany( + { + group_id: this.group_id + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal + ); + } + + protected getDataBucketChangesImpl( + options: GetCheckpointChangesOptions + ): Promise> { + return getDataBucketChangesV1(this.versionContext, options); + } + + protected getParameterBucketChangesImpl( + options: GetCheckpointChangesOptions + ): Promise> { + return getParameterBucketChangesV1(this.versionContext, options); + } +} + +export async function getParameterSetsV1( + ctx: MongoSyncBucketStorageContext, + checkpoint: MongoSyncBucketStorageCheckpoint, + lookups: ScopedParameterLookup[] +): Promise { + return ctx.db.client.withSession({ snapshot: true }, async (session) => { + setSessionSnapshotTime(session, checkpoint.snapshotTime); + const lookupFilter = lookups.map((lookup) => { + return storage.serializeLookup(lookup); + }); + const rows = await ctx.db.parameterIndexV1 + .aggregate( + [ + { + $match: { + 'key.g': ctx.group_id, + lookup: { $in: lookupFilter }, + _id: { $lte: checkpoint.checkpoint } + } + }, + { + $sort: { + _id: -1 + } + }, + { + $group: { + _id: { key: '$key', lookup: '$lookup' }, + bucket_parameters: { + $first: '$bucket_parameters' + } + } + } + ], + { + session, + readConcern: 'snapshot', + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + } + ) + .toArray() + .catch((e) => { + throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries'); + }); + const groupedParameters = rows.map((row) => { + return row.bucket_parameters; + }); + return groupedParameters.flat(); + }); +} + +export async function* getBucketDataBatchV1( + ctx: MongoSyncBucketStorageContext, + checkpoint: utils.InternalOpId, + dataBuckets: storage.BucketDataRequest[], + options?: storage.BucketDataBatchOptions +): AsyncIterable { + if (dataBuckets.length == 0) { + return; + } + let filters: mongo.Filter[] = []; + const bucketMap = new Map(dataBuckets.map((request) => [request.bucket, request.start])); + + if (checkpoint == null) { + throw new Error('checkpoint is null'); + } + const end = checkpoint; + for (let { bucket: name, start } of dataBuckets) { + filters.push({ + _id: { + $gt: { + g: ctx.group_id, + b: name, + o: start + }, + $lte: { + g: ctx.group_id, + b: name, + o: end as any + } + } + }); + } + + const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; + const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; + + const cursor = ctx.db.bucket_data.find( + { + $or: filters + }, + { + session: undefined, + sort: { _id: 1 }, + limit: batchLimit, + batchSize: batchLimit + 1, + raw: true, + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + } + ) as unknown as mongo.FindCursor; + + let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => { + throw lib_mongo.mapQueryError(e, 'while reading bucket data'); + }); + if (data.length == batchLimit) { + batchHasMore = true; + } + + let chunkSizeBytes = 0; + let currentChunk: utils.SyncBucketData | null = null; + let targetOp: InternalOpId | null = null; + + for (let rawData of data) { + const row = loadBucketDataDocumentV1( + bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV1 + ); + const bucket = row.bucketKey.bucket; + + if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) { + let start: ProtocolOpId | undefined = undefined; + if (currentChunk != null) { + if (currentChunk.bucket == bucket) { + currentChunk.has_more = true; + start = currentChunk.next_after; + } + + const yieldChunk = currentChunk; + currentChunk = null; + chunkSizeBytes = 0; + yield { chunkData: yieldChunk, targetOp: targetOp }; + targetOp = null; + } + + if (start == null) { + const startOpId = bucketMap.get(bucket); + if (startOpId == null) { + throw new Error(`data for unexpected bucket: ${bucket}`); + } + start = internalToExternalOpId(startOpId); + } + currentChunk = { + bucket, + after: start, + has_more: false, + data: [], + next_after: start + }; + targetOp = null; + } + + const entry = mapOpEntry(row); + + if (row.target_op != null && (targetOp == null || row.target_op > targetOp)) { + targetOp = row.target_op; + } + + currentChunk.data.push(entry); + currentChunk.next_after = entry.op_id; + chunkSizeBytes += rawData.byteLength; + } + + if (currentChunk != null) { + const yieldChunk = currentChunk; + yieldChunk.has_more = batchHasMore; + yield { chunkData: yieldChunk, targetOp: targetOp }; + } +} + +export async function getDataBucketChangesV1( + ctx: MongoSyncBucketStorageContext, + options: GetCheckpointChangesOptions +): Promise> { + const limit = 1000; + const bucketStateUpdates = await ctx.db.bucketStateV1 + .find( + { + '_id.g': ctx.group_id, + last_op: { $gt: options.lastCheckpoint.checkpoint } + }, + { + projection: { + '_id.b': 1 + }, + limit: limit + 1, + batchSize: limit + 2, + singleBatch: true + } + ) + .toArray(); + + const buckets = bucketStateUpdates.map((doc) => doc._id.b); + const invalidateDataBuckets = buckets.length > limit; + + return { + invalidateDataBuckets, + updatedDataBuckets: invalidateDataBuckets ? new Set() : new Set(buckets) + }; +} + +export async function getParameterBucketChangesV1( + ctx: MongoSyncBucketStorageContext, + options: GetCheckpointChangesOptions +): Promise> { + const limit = 1000; + const parameterUpdates = await ctx.db.parameterIndexV1 + .find( + { + _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint }, + 'key.g': ctx.group_id + }, + { + projection: { + lookup: 1 + }, + limit: limit + 1, + batchSize: limit + 2, + singleBatch: true + } + ) + .toArray(); + const invalidateParameterUpdates = parameterUpdates.length > limit; + + return { + invalidateParameterBuckets: invalidateParameterUpdates, + updatedParameterLookups: invalidateParameterUpdates + ? new Set() + : new Set(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookup(p.lookup)))) + }; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts new file mode 100644 index 000000000..e300bc29a --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts @@ -0,0 +1,230 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import * as bson from 'bson'; + +import { BucketDataSource } from '@powersync/service-sync-rules'; +import { mongoTableId } from '../../../utils/util.js'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import { EMPTY_DATA } from '../MongoBucketBatchShared.js'; +import { + BucketStateUpdate, + PersistedBatch, + SaveParameterDataOptions, + UpsertCurrentDataOptions +} from '../common/PersistedBatch.js'; +import { LEGACY_BUCKET_DATA_DEFINITION_ID, LEGACY_BUCKET_PARAMETER_INDEX_ID, SourceKey } from '../models.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; +import { + BucketParameterDocument, + BucketStateDocumentV1, + CurrentDataDocument, + serializeBucketDataV1, + taggedBucketParameterDocumentToV1 +} from './models.js'; + +export class PersistedBatchV1 extends PersistedBatch { + declare protected readonly db: VersionedPowerSyncMongoV1; + + currentData: mongo.AnyBulkWriteOperation[] = []; + + protected checkDefinitionId(_definitionId: BucketDefinitionId | null): BucketDefinitionId { + // V1 storage doesn't persist the id, and we don't use it. + return LEGACY_BUCKET_DATA_DEFINITION_ID; + } + + protected getBucketDefinitionId(_bucketSource: BucketDataSource): BucketDefinitionId { + return LEGACY_BUCKET_DATA_DEFINITION_ID; + } + + saveParameterData(data: SaveParameterDataOptions) { + const { sourceTable, sourceKey, evaluated } = data; + const remaining_lookups = new Map(); + + for (let lookup of data.existing_lookups) { + if (lookup.indexId != null) { + throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); + } + remaining_lookups.set(lookup.lookup.toString('base64'), lookup.lookup); + } + + for (let result of evaluated) { + const binLookup = storage.serializeLookup(result.lookup); + remaining_lookups.delete(binLookup.toString('base64')); + + const op_id = data.op_seq.next(); + this.debugLastOpId = op_id; + const values: BucketParameterDocument = { + _id: op_id, + key: { + g: this.group_id, + t: mongoTableId(sourceTable.id), + k: sourceKey + }, + lookup: binLookup, + bucket_parameters: result.bucketParameters + }; + this.bucketParameters.push({ + ...values, + index: LEGACY_BUCKET_PARAMETER_INDEX_ID + }); + + this.currentSize += 200; + } + + for (let lookup of remaining_lookups.values()) { + const op_id = data.op_seq.next(); + this.debugLastOpId = op_id; + const values: BucketParameterDocument = { + _id: op_id, + key: { + g: this.group_id, + t: mongoTableId(sourceTable.id), + k: sourceKey + }, + lookup, + bucket_parameters: [] + }; + this.bucketParameters.push({ + ...values, + index: LEGACY_BUCKET_PARAMETER_INDEX_ID + }); + + this.currentSize += 200; + } + } + + hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId) { + this.currentData.push({ + deleteOne: { + filter: { _id: this.currentDataId(sourceTableId, replicaId) } + } + }); + this.currentSize += 50; + } + + softDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId, _checkpointGreaterThan: bigint) { + this.hardDeleteCurrentData(sourceTableId, replicaId); + } + + upsertCurrentData(values: UpsertCurrentDataOptions) { + const buckets = values.buckets.map((bucket) => { + if (bucket.definitionId != null) { + throw new ReplicationAssertionError('Unexpected v3 bucket when incrementalReprocessing is disabled'); + } + return { + bucket: bucket.bucket, + table: bucket.table, + id: bucket.id + }; + }); + const lookups = values.lookups.map((lookup) => { + if (lookup.indexId != null) { + throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); + } + return lookup.lookup; + }); + + this.currentData.push({ + updateOne: { + filter: { _id: this.currentDataId(values.sourceTableId, values.replicaId) }, + update: { + $set: { + data: values.data ?? EMPTY_DATA, + buckets, + lookups + } + }, + upsert: true + } + }); + this.currentSize += (values.data?.length() ?? 0) + 100; + } + + protected get currentDataCount() { + return this.currentData.length; + } + + protected async flushBucketData(session: mongo.ClientSession) { + await this.db.bucketDataV1.bulkWrite( + this.bucketData.map((document) => ({ + insertOne: { + document: serializeBucketDataV1(document) + } + })), + { + session, + ordered: false + } + ); + } + + protected async flushBucketParameters(session: mongo.ClientSession) { + await this.db.parameterIndexV1.bulkWrite( + this.bucketParameters.map((document) => ({ + insertOne: { + document: taggedBucketParameterDocumentToV1(document) + } + })), + { + session, + ordered: false + } + ); + } + + protected async flushCurrentData(session: mongo.ClientSession) { + if (this.currentData.length == 0) { + return; + } + + await this.db.sourceRecordsV1.bulkWrite(this.currentData, { + session, + ordered: true + }); + } + + protected async flushBucketStates(session: mongo.ClientSession) { + await this.db.bucketStateV1.bulkWrite(this.getBucketStateUpdates(), { + session, + ordered: false + }); + } + + protected resetCurrentData() { + this.currentData = []; + } + + private getBucketStateUpdates(): mongo.AnyBulkWriteOperation[] { + return Array.from(this.bucketStates.values()).map((state: BucketStateUpdate) => { + return { + updateOne: { + filter: { + _id: { + g: this.group_id, + b: state.bucket + } + }, + update: { + $set: { + last_op: state.lastOp + }, + $inc: { + 'estimate_since_compact.count': state.incrementCount, + 'estimate_since_compact.bytes': state.incrementBytes + } + }, + upsert: true + } + } satisfies mongo.AnyBulkWriteOperation; + }); + } + + private currentDataId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): SourceKey { + return { + g: this.group_id, + t: sourceTableId, + k: replicaId + }; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/SingleBucketStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/SingleBucketStoreV1.ts new file mode 100644 index 000000000..c0300bc16 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/SingleBucketStoreV1.ts @@ -0,0 +1,74 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { InternalOpId } from '@powersync/service-core'; +import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js'; +import { + BucketDataDocumentGeneric, + BucketDataDocumentGenericId, + SingleBucketStore +} from '../common/SingleBucketStore.js'; +import { BucketDataProperties } from '../models.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; +import { BucketDataDocumentV1, BucketDataKeyV1, serializeBucketDataV1 } from './models.js'; + +export class SingleBucketStoreV1 implements SingleBucketStore { + public readonly collection: mongo.Collection; + + constructor( + private db: VersionedPowerSyncMongoV1, + public readonly key: BucketKey + ) { + this.collection = db.bucketDataV1 as unknown as mongo.Collection; + } + + docId(o: InternalOpId): BucketDataDocumentGenericId { + // `satisfies BucketDataKeyV1` checks that we use the correct type for V1 storage + // `as anyt` is to allow casting to the interface virtual type + return { + g: this.key.replicationStreamId, + b: this.key.bucket, + o + } satisfies BucketDataKeyV1 as any; + } + + get minId(): BucketDataDocumentGenericId { + return { + g: this.key.replicationStreamId, + b: this.key.bucket, + o: new mongo.MinKey() + } as any; + } + + get maxId(): BucketDataDocumentGenericId { + return { + g: this.key.replicationStreamId, + b: this.key.bucket, + o: new mongo.MaxKey() + } as any; + } + + toPersistedDocument(source: Omit): BucketDataDocumentGeneric { + return serializeBucketDataV1({ bucketKey: this.key, ...source }) as unknown as BucketDataDocumentGeneric; + } + + fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc { + const document = doc as unknown as BucketDataDocumentV1; + const { _id, ...rest } = document; + return { + bucketKey: this.key, + o: _id.o, + ...rest + }; + } + + fromPartialPersistedDocument( + doc: Pick + ): Pick { + const document = doc as Pick; + const { _id, ...rest } = document; + return { + bucketKey: this.key, + o: _id.o, + ...rest + } as Pick; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts new file mode 100644 index 000000000..c5f990159 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts @@ -0,0 +1,156 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { Logger } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; +import { idPrefixFilter } from '../../../utils/util.js'; +import { cacheKey } from '../OperationBatch.js'; +import { + LoadedSourceRecord, + SourceRecordLookupEntry, + SourceRecordLookupState, + SourceRecordStore +} from '../common/SourceRecordStore.js'; +import { SourceKey } from '../models.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; +import { CurrentDataDocument } from './models.js'; + +export class SourceRecordStoreV1 implements SourceRecordStore { + constructor( + private readonly db: VersionedPowerSyncMongoV1, + private readonly groupId: number + ) {} + + mapEvaluatedBuckets(evaluated: EvaluatedRow[]): LoadedSourceRecord['buckets'] { + return evaluated.map((entry) => ({ + definitionId: null, + bucket: entry.bucket, + table: entry.table, + id: entry.id + })); + } + + mapParameterLookups(paramEvaluated: EvaluatedParameters[]): SourceRecordLookupState[] { + return paramEvaluated.map((entry) => ({ + indexId: null, + lookup: storage.serializeLookup(entry.lookup) + })); + } + + private createId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): SourceKey { + return { + g: this.groupId, + t: sourceTableId, + k: replicaId + } satisfies SourceKey; + } + + private createLoadedDocument( + sourceTableId: bson.ObjectId, + id: SourceKey, + data: bson.Binary | null, + buckets: CurrentDataDocument['buckets'], + lookups: CurrentDataDocument['lookups'] + ): LoadedSourceRecord { + return { + sourceTableId, + replicaId: id.k, + data, + buckets: buckets.map((bucket) => ({ + definitionId: null, + bucket: bucket.bucket, + table: bucket.table, + id: bucket.id + })), + lookups: lookups.map((lookup) => ({ + indexId: null, + lookup + })), + cacheKey: cacheKey(sourceTableId, id.k) + }; + } + + async loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise> { + const sizes = new Map(); + const sizeCursor: mongo.AggregationCursor = + this.db.sourceRecordsV1.aggregate( + [ + { + $match: { + _id: { + $in: entries.map((entry) => this.createId(entry.sourceTableId, entry.replicaId) as SourceKey) + } + } + }, + { + $project: { + _id: 1, + size: { $bsonSize: '$$ROOT' } + } + } + ], + { session } + ); + for await (const doc of sizeCursor.stream()) { + sizes.set(cacheKey(doc._id.t, doc._id.k), doc.size); + } + return sizes; + } + + async loadDocuments( + session: mongo.ClientSession, + entries: SourceRecordLookupEntry[], + idsOnly: boolean + ): Promise> { + const documents = new Map(); + const projection = idsOnly ? { _id: 1 } : undefined; + const cursor = this.db.sourceRecordsV1.find( + { + _id: { + $in: entries.map((entry) => this.createId(entry.sourceTableId, entry.replicaId) as SourceKey) + } + }, + { session, projection } + ); + for await (const doc of cursor.stream()) { + const loaded = this.createLoadedDocument( + doc._id.t, + doc._id, + idsOnly ? null : doc.data, + idsOnly ? [] : doc.buckets, + idsOnly ? [] : doc.lookups + ); + documents.set(loaded.cacheKey, loaded); + } + return documents; + } + + async loadTruncateBatch( + session: mongo.ClientSession, + sourceTableId: bson.ObjectId, + limit: number + ): Promise { + const cursor = this.db.sourceRecordsV1.find( + { + _id: idPrefixFilter({ g: this.groupId, t: sourceTableId }, ['k']), + pending_delete: { $exists: false } + }, + { + projection: { + _id: 1, + buckets: 1, + lookups: 1 + }, + limit, + session + } + ); + return (await cursor.toArray()).map((doc) => + this.createLoadedDocument(sourceTableId, doc._id, null, doc.buckets, doc.lookups) + ); + } + + async postCommitCleanup(_lastCheckpoint: bigint, _logger: Logger): Promise { + // No-op for V1. + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts new file mode 100644 index 000000000..0758dd4f2 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts @@ -0,0 +1,28 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js'; +import { CommonSourceTableDocument } from '../models.js'; +import { BucketDataDocumentV1, BucketParameterDocument, BucketStateDocumentV1, CurrentDataDocument } from './models.js'; + +export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo { + get sourceRecordsV1(): mongo.Collection { + return this.upstream.current_data; + } + + get bucketStateV1(): mongo.Collection { + return this.upstream.bucket_state; + } + + commonSourceTables(_replicationStreamId: number): mongo.Collection { + return this.upstream.source_tables as any as mongo.Collection; + } + + async initializeStreamStorage(_replicationStreamId: number): Promise {} + + get bucketDataV1(): mongo.Collection { + return this.upstream.bucket_data; + } + + get parameterIndexV1(): mongo.Collection { + return this.upstream.bucket_parameters; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts new file mode 100644 index 000000000..e5567ce4c --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts @@ -0,0 +1,84 @@ +import * as bson from 'bson'; +import { BucketDataDoc } from '../common/BucketDataDoc.js'; +import { + BucketDataDocumentBase, + BucketParameterDocumentBase, + BucketStateDocumentBase, + CurrentBucket, + LEGACY_BUCKET_DATA_DEFINITION_ID, + SourceKey, + SourceTableDocument, + TaggedBucketParameterDocument +} from '../models.js'; + +export interface BucketDataKeyV1 { + /** group_id */ + g: number; + /** bucket name */ + b: string; + /** op_id */ + o: bigint; +} + +export interface CurrentDataDocument { + _id: SourceKey; + data: bson.Binary; + buckets: CurrentBucket[]; + lookups: bson.Binary[]; +} + +export interface BucketParameterDocument extends BucketParameterDocumentBase {} + +export interface BucketDataDocumentV1 extends BucketDataDocumentBase { + _id: BucketDataKeyV1; +} + +export function serializeBucketDataV1(document: BucketDataDoc): BucketDataDocumentV1 { + const { bucketKey, o } = document; + return { + _id: { + g: bucketKey.replicationStreamId, + b: bucketKey.bucket, + o: o + }, + // List fields directly, so that we don't accidentally persist any unknown fields + op: document.op, + source_table: document.source_table, + source_key: document.source_key, + table: document.table, + row_id: document.row_id, + checksum: document.checksum, + data: document.data, + target_op: document.target_op + }; +} + +export function loadBucketDataDocumentV1(doc: BucketDataDocumentV1): BucketDataDoc { + const { _id, ...rest } = doc; + return { + bucketKey: { + replicationStreamId: _id.g, + definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID, + bucket: _id.b + }, + o: _id.o, + ...rest + }; +} + +export function taggedBucketParameterDocumentToV1(document: TaggedBucketParameterDocument): BucketParameterDocument { + const { index: _index, ...rest } = document; + return rest as BucketParameterDocument; +} + +export interface SourceTableDocumentV1 extends SourceTableDocument { + group_id: number; +} + +export interface BucketStateDocumentV1 extends BucketStateDocumentBase { + _id: BucketStateDocumentBase['_id'] & { + g: number; + }; +} + +export type BucketStateDocument = BucketStateDocumentV1; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts new file mode 100644 index 000000000..115548c36 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts @@ -0,0 +1,44 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { storage } from '@powersync/service-core'; +import { mongoTableId } from '../../../utils/util.js'; +import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js'; +import { PersistedBatch } from '../common/PersistedBatch.js'; +import { SourceRecordStore } from '../common/SourceRecordStore.js'; +import { PersistedBatchV3 } from './PersistedBatchV3.js'; +import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; + +export class MongoBucketBatchV3 extends MongoBucketBatch { + declare public readonly db: VersionedPowerSyncMongoV3; + + private readonly store: SourceRecordStore; + + constructor(options: MongoBucketBatchOptions) { + super(options); + this.store = new SourceRecordStoreV3(this.db, this.group_id, this.mapping); + } + + protected createPersistedBatch(writtenSize: number): PersistedBatch { + return new PersistedBatchV3(this.db, this.group_id, this.mapping, writtenSize, { + logger: this.logger + }); + } + + protected get sourceRecordStore(): SourceRecordStore { + return this.store; + } + + protected async cleanupDroppedSourceTables(sourceTables: storage.SourceTable[]) { + for (const table of sourceTables) { + await this.db + .sourceRecordsV3(this.group_id, mongoTableId(table.id)) + .drop() + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { + return; + } + throw error; + }); + } + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts new file mode 100644 index 000000000..88e0b42fa --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -0,0 +1,120 @@ +import { + bson, + BucketChecksum, + FetchPartialBucketChecksum, + InternalOpId, + PartialChecksumMap, + PartialOrFullChecksum +} from '@powersync/service-core'; +import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; +import { + emptyChecksumForRequest, + FetchPartialBucketChecksumV3, + MongoChecksumOptions, + MongoChecksums +} from '../MongoChecksums.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; + +export class MongoChecksumsV3 extends MongoChecksums { + declare protected readonly db: VersionedPowerSyncMongoV3; + private readonly mapping: BucketDefinitionMapping; + + constructor(db: VersionedPowerSyncMongoV3, group_id: number, options: MongoChecksumOptions) { + super(db, group_id, options); + this.mapping = options.mapping!; + } + + private normalizeBatch(batch: FetchPartialBucketChecksum[]): FetchPartialBucketChecksumV3[] { + return batch.map((request) => ({ + bucket: request.bucket, + definitionId: this.mapping.bucketSourceId(request.source), + start: request.start, + end: request.end + })); + } + + async computePartialChecksumsDirectByDefinition(batch: FetchPartialBucketChecksumV3[]): Promise { + const results = new Map(); + const requestsByDefinition = new Map(); + + for (const request of batch) { + const existing = requestsByDefinition.get(request.definitionId) ?? []; + existing.push(request); + requestsByDefinition.set(request.definitionId, existing); + } + + for (const [definitionId, requests] of requestsByDefinition.entries()) { + const groupResults = await this.computePartialChecksumsForCollection( + requests, + this.db.bucketDataV3(this.group_id, definitionId), + createV3BucketFilter + ); + for (const checksum of groupResults.values()) { + results.set(checksum.bucket, checksum); + } + } + + return new Map( + batch.map((request) => [request.bucket, results.get(request.bucket) ?? emptyChecksumForRequest(request)]) + ); + } + + protected async fetchPreStates( + batch: FetchPartialBucketChecksum[] + ): Promise> { + const preFilters = this.normalizeBatch(batch) + .filter((request) => request.start == null) + .map((request) => ({ + _id: { + d: request.definitionId, + b: request.bucket + }, + 'compacted_state.op_id': { $exists: true, $lte: request.end } + })); + + const preStates = new Map(); + if (preFilters.length == 0) { + return preStates; + } + + const states = await this.db + .bucketStateV3(this.group_id) + .find({ + $or: preFilters + }) + .toArray(); + + for (const state of states) { + const compactedState = state.compacted_state!; + preStates.set(state._id.b, { + opId: compactedState.op_id, + checksum: { + bucket: state._id.b, + checksum: Number(compactedState.checksum), + count: compactedState.count + } + }); + } + + return preStates; + } + + protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { + return this.computePartialChecksumsDirectByDefinition(this.normalizeBatch(batch)); + } +} + +function createV3BucketFilter(request: Pick) { + return { + _id: { + $gt: { + b: request.bucket, + o: request.start ?? new bson.MinKey() + }, + $lte: { + b: request.bucket, + o: request.end + } + } + }; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts new file mode 100644 index 000000000..b9641ca8d --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -0,0 +1,107 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import { SingleBucketStore } from '../common/SingleBucketStore.js'; +import { BucketStateDocumentBase } from '../models.js'; +import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; +import { BucketStateDocumentV3 } from './models.js'; +import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; +import { SingleBucketStoreV3 } from './SingleBucketStoreV3.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; + +export class MongoCompactorV3 extends MongoCompactor { + declare protected readonly db: VersionedPowerSyncMongoV3; + declare protected readonly storage: MongoSyncBucketStorageV3; + + public async *dirtyBucketBatches(options: { + minBucketChanges: number; + minChangeRatio: number; + }): AsyncGenerator { + if (options.minBucketChanges <= 0) { + throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + } + // Same scan strategy as V1, but with the V3 bucket_state key shape. + yield* this.dirtyBucketBatchesForCollection( + this.db.bucketStateV3(this.group_id), + { d: new mongo.MinKey() as any, b: new mongo.MinKey() as any }, + { d: new mongo.MaxKey() as any, b: new mongo.MaxKey() as any }, + options, + (bucketState) => (bucketState as BucketStateDocumentV3)._id.d + ); + } + + public async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { + if (options.minBucketChanges <= 0) { + throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + } + return this.dirtyBucketBatchForChecksumsForCollection( + this.db.bucketStateV3(this.group_id), + { + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + }, + (bucketState) => (bucketState as BucketStateDocumentV3)._id.d + ); + } + + protected async writeBucketStateUpdates(): Promise { + await this.db + .bucketStateV3(this.group_id) + .bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], { ordered: false }); + } + + protected async computeChecksumsForBuckets( + buckets: Pick[] + ): Promise { + return this.storage.checksums.computePartialChecksumsDirectByDefinition( + buckets.map(({ bucket, definitionId }) => { + if (definitionId == null) { + throw new ServiceAssertionError(`Missing definitionId for V3 bucket checksum update on bucket ${bucket}`); + } + return { + bucket, + definitionId, + end: this.maxOpId + }; + }) + ); + } + + protected bucketStateFilter( + bucket: string, + definitionId: BucketDefinitionId | null + ): mongo.Filter { + if (definitionId == null) { + throw new ServiceAssertionError(`Missing definitionId for V3 bucket state filter on bucket ${bucket}`); + } + return { + _id: { + d: definitionId, + b: bucket + } + }; + } + + protected async getBucketDataContext( + bucket: string, + definitionId: BucketDefinitionId | null + ): Promise { + if (definitionId == null) { + // Not the _most_ efficient approach, but this is not used often + const allDefinitionIds = this.storage.mapping.allBucketDefinitionIds(); + if (allDefinitionIds.length == 0) { + return null; + } + const potentialIds = allDefinitionIds.map((definitionId) => ({ d: definitionId, b: bucket })); + const bucketState = await this.db.bucketStateV3(this.group_id).findOne({ + _id: { $in: potentialIds } + }); + if (bucketState == null) { + return null; + } + definitionId = bucketState._id.d; + } + + return new SingleBucketStoreV3(this.db, { bucket, definitionId, replicationStreamId: this.group_id }); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts new file mode 100644 index 000000000..433c0ac7b --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts @@ -0,0 +1,24 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; + +export class MongoParameterCompactorV3 extends MongoParameterCompactor { + declare protected readonly db: VersionedPowerSyncMongoV3; + + protected async getCollections(): Promise[]> { + const collections = await this.db.listParameterIndexCollectionsV3(this.group_id); + return collections.map((collection) => collection.collection as unknown as mongo.Collection); + } + + protected collectionFilter(): mongo.Document { + return {}; + } + + protected deleteFilter(doc: mongo.Document): mongo.Document { + return { + lookup: doc.lookup, + _id: { $lte: doc._id }, + key: doc.key + }; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterLookupV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterLookupV3.ts new file mode 100644 index 000000000..b787608ce --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterLookupV3.ts @@ -0,0 +1,12 @@ +import { deserializeParameterLookup } from '@powersync/service-core'; +import { ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; +import { ParameterIndexId } from '../BucketDefinitionMapping.js'; + +export function serializeParameterLookupV3(lookup: ScopedParameterLookup): bson.Binary { + return new bson.Binary(bson.serialize({ l: lookup.values.slice(2) })); +} + +export function deserializeParameterLookupV3(lookup: bson.Binary, indexId: ParameterIndexId): SqliteJsonValue[] { + return [indexId, '', ...deserializeParameterLookup(lookup)]; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts new file mode 100644 index 000000000..a280a148b --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -0,0 +1,534 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; +import { + CheckpointChanges, + GetCheckpointChangesOptions, + InternalOpId, + internalToExternalOpId, + ProtocolOpId, + storage, + utils +} from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; +import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; +import { MongoBucketStorage } from '../../MongoBucketStorage.js'; +import { + MongoSyncBucketStorageCheckpoint, + MongoSyncBucketStorageContext +} from '../common/MongoSyncBucketStorageContext.js'; +import { CommonSourceTableDocument } from '../models.js'; +import { MongoBucketBatchOptions } from '../MongoBucketBatch.js'; +import { MongoChecksums } from '../MongoChecksums.js'; +import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; +import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; +import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; +import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; +import { BucketDataDocumentV3, BucketParameterDocumentV3, loadBucketDataDocumentV3 } from './models.js'; +import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; +import { MongoChecksumsV3 } from './MongoChecksumsV3.js'; +import { MongoCompactorV3 } from './MongoCompactorV3.js'; +import { MongoParameterCompactorV3 } from './MongoParameterCompactorV3.js'; +import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; + +export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { + // Declare types to be more specific + declare readonly db: VersionedPowerSyncMongoV3; + declare readonly checksums: MongoChecksumsV3; + + constructor( + factory: MongoBucketStorage, + group_id: number, + sync_rules: MongoPersistedSyncRulesContent, + slot_name: string, + writeCheckpointMode: storage.WriteCheckpointMode | undefined, + options: MongoSyncBucketStorageOptions + ) { + super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); + } + + protected async initializeVersionStorage(): Promise { + const mapping = this.mapping; + for (let source of mapping.allBucketDefinitionIds()) { + const collection = this.db.bucketDataV3(this.group_id, source).collectionName; + await this.db.db + .createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } }) + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceExists') { + return; + } + throw error; + }); + } + for (let indexId of mapping.allParameterIndexIds()) { + await this.db.parameterIndexV3(this.group_id, indexId).createIndex( + { + lookup: 1, + key: 1, + _id: -1 + }, + { + name: 'lookup_op_id' + } + ); + } + } + + protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums { + return new MongoChecksumsV3(this.db, this.group_id, { + ...options.checksumOptions, + storageConfig: options?.storageConfig, + mapping: this.sync_rules.mapping + }); + } + + createMongoCompactor(options: MongoCompactOptions): MongoCompactor { + return new MongoCompactorV3(this, this.db, options); + } + + protected createMongoParameterCompactor( + checkpoint: InternalOpId, + options: storage.CompactOptions + ): MongoParameterCompactor { + return new MongoParameterCompactorV3(this.db, this.group_id, checkpoint, options); + } + + protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch { + return new MongoBucketBatchV3(batchOptions); + } + + protected sourceTableBaseId(): Partial { + return {}; + } + + protected augmentCreatedSourceTableDocument( + createDoc: CommonSourceTableDocument, + options: storage.ResolveTableOptions, + candidateSourceTable: storage.SourceTable + ): void { + const bucketDataSourceIds = options.sync_rules.definition.bucketDataSources + .filter((source) => source.tableSyncsData(candidateSourceTable)) + .map((source) => this.mapping.bucketSourceId(source)); + const parameterLookupSourceIds = options.sync_rules.definition.bucketParameterLookupSources + .filter((source) => source.tableSyncsParameters(candidateSourceTable)) + .map((source) => this.mapping.parameterLookupId(source)); + + Object.assign(createDoc, { + bucket_data_source_ids: bucketDataSourceIds, + parameter_lookup_source_ids: parameterLookupSourceIds + }); + } + + protected async initializeResolvedSourceRecords(sourceTableId: bson.ObjectId): Promise { + await this.db.initializeSourceRecordsCollection(this.group_id, sourceTableId); + } + + protected override get versionContext(): MongoSyncBucketStorageContext { + return { + db: this.db, + group_id: this.group_id, + mapping: this.mapping + }; + } + + protected getParameterSetsImpl( + checkpoint: MongoSyncBucketStorageCheckpoint, + lookups: ScopedParameterLookup[] + ): Promise { + return getParameterSetsV3(this.versionContext, checkpoint, lookups); + } + + protected getBucketDataBatchImpl( + checkpoint: utils.InternalOpId, + dataBuckets: storage.BucketDataRequest[], + options?: storage.BucketDataBatchOptions + ): AsyncIterable { + return getBucketDataBatchV3(this.versionContext, checkpoint, dataBuckets, options); + } + + protected async clearBucketData(_signal?: AbortSignal): Promise { + for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) { + await collection.drop(); + } + } + + protected async clearParameterIndexes(_signal?: AbortSignal): Promise { + for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) { + await collection.collection.drop(); + } + } + + protected async clearSourceRecords(_signal?: AbortSignal): Promise { + for (const collection of await this.db.listSourceRecordCollectionsV3(this.group_id)) { + await collection.drop(); + } + } + + protected async clearBucketState(_signal?: AbortSignal): Promise { + await this.db + .bucketStateV3(this.group_id) + .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }) + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { + return; + } + throw error; + }); + } + + protected async clearSourceTables(_signal?: AbortSignal): Promise { + await this.db + .sourceTablesV3(this.group_id) + .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }) + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { + return; + } + throw error; + }); + } + + protected getDataBucketChangesImpl( + options: GetCheckpointChangesOptions + ): Promise> { + return getDataBucketChangesV3(this.versionContext, options); + } + + protected getParameterBucketChangesImpl( + options: GetCheckpointChangesOptions + ): Promise> { + return getParameterBucketChangesV3(this.versionContext, options); + } +} + +export async function getParameterSetsV3( + ctx: MongoSyncBucketStorageContext, + checkpoint: MongoSyncBucketStorageCheckpoint, + lookups: ScopedParameterLookup[] +): Promise { + return ctx.db.client.withSession({ snapshot: true }, async (session) => { + setSessionSnapshotTime(session, checkpoint.snapshotTime); + + const buildLookupPipeline = ( + lookup: ScopedParameterLookup + ): { + collection: mongo.Collection; + pipeline: mongo.Document[]; + } => { + const indexId = lookup.indexId; + const collection = ctx.db.parameterIndexV3(ctx.group_id, indexId); + const lookupFilter = serializeParameterLookupV3(lookup); + return { + collection, + pipeline: [ + { + $match: { + lookup: lookupFilter, + _id: { $lte: checkpoint.checkpoint } + } + }, + { + $sort: { + key: 1, + _id: -1 + } + }, + { + $group: { + _id: { + key: '$key' + }, + bucket_parameters: { + $first: '$bucket_parameters' + } + } + }, + { + $project: { + _id: 0, + bucket_parameters: 1 + } + } + ] + }; + }; + + const [firstLookup, ...remainingLookups] = lookups; + const firstQuery = firstLookup == null ? null : buildLookupPipeline(firstLookup); + if (firstQuery == null) { + return []; + } + + const pipeline: mongo.Document[] = [ + ...firstQuery.pipeline, + ...remainingLookups.map((lookup) => { + const query = buildLookupPipeline(lookup); + return { + $unionWith: { + coll: query.collection.collectionName, + pipeline: query.pipeline + } + }; + }) + ]; + + const rows = await firstQuery.collection + .aggregate<{ bucket_parameters: SqliteJsonRow[] }>(pipeline, { + session, + readConcern: 'snapshot', + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + }) + .toArray() + .catch((e) => { + throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries'); + }); + + return rows.flatMap((row) => row.bucket_parameters); + }); +} + +export async function* getBucketDataBatchV3( + ctx: MongoSyncBucketStorageContext, + checkpoint: utils.InternalOpId, + dataBuckets: storage.BucketDataRequest[], + options?: storage.BucketDataBatchOptions +): AsyncIterable { + if (dataBuckets.length == 0) { + return; + } + + if (checkpoint == null) { + throw new Error('checkpoint is null'); + } + + const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; + const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; + const end = checkpoint; + let remainingLimit = batchLimit; + + const requestsByDefinition = new Map(); + for (const request of dataBuckets) { + const definitionId = ctx.mapping.bucketSourceId(request.source); + const requests = requestsByDefinition.get(definitionId) ?? []; + requests.push(request); + requestsByDefinition.set(definitionId, requests); + } + + const definitionGroups = Array.from(requestsByDefinition.entries()); + for (let groupIndex = 0; groupIndex < definitionGroups.length && remainingLimit > 0; groupIndex++) { + const [definitionId, requests] = definitionGroups[groupIndex]; + const hasLaterDefinitionGroups = groupIndex < definitionGroups.length - 1; + const bucketMap = new Map(requests.map((request) => [request.bucket, request.start])); + const filters: mongo.Filter[] = Array.from(bucketMap.entries()).map(([bucket, start]) => ({ + _id: { + $gt: { + b: bucket, + o: start + }, + $lte: { + b: bucket, + o: end as any + } + } + })); + + const cursor = ctx.db.bucketDataV3(ctx.group_id, definitionId).find( + { + $or: filters + }, + { + session: undefined, + sort: { _id: 1 }, + limit: remainingLimit, + batchSize: remainingLimit + 1, + raw: true, + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + } + ) as unknown as mongo.FindCursor; + + let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => { + throw lib_mongo.mapQueryError(e, 'while reading bucket data'); + }); + if (data.length == remainingLimit) { + batchHasMore = true; + } + if (data.length == 0) { + continue; + } + + remainingLimit -= data.length; + + let chunkSizeBytes = 0; + let currentChunk: utils.SyncBucketData | null = null; + let targetOp: InternalOpId | null = null; + + for (let rawData of data) { + const row = loadBucketDataDocumentV3( + { replicationStreamId: ctx.group_id, definitionId }, + bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV3 + ); + const bucket = row.bucketKey.bucket; + + if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) { + let start: ProtocolOpId | undefined = undefined; + if (currentChunk != null) { + if (currentChunk.bucket == bucket) { + currentChunk.has_more = true; + start = currentChunk.next_after; + } + + const yieldChunk = currentChunk; + currentChunk = null; + chunkSizeBytes = 0; + yield { chunkData: yieldChunk, targetOp: targetOp }; + targetOp = null; + } + + if (start == null) { + const startOpId = bucketMap.get(bucket); + if (startOpId == null) { + throw new Error(`data for unexpected bucket: ${bucket}`); + } + start = internalToExternalOpId(startOpId); + } + currentChunk = { + bucket, + after: start, + has_more: false, + data: [], + next_after: start + }; + } + + const entry = mapOpEntry(row); + if (row.target_op != null && (targetOp == null || row.target_op > targetOp)) { + targetOp = row.target_op; + } + + currentChunk.data.push(entry); + currentChunk.next_after = entry.op_id; + chunkSizeBytes += rawData.byteLength; + } + + if (currentChunk != null) { + const yieldChunk = currentChunk; + yieldChunk.has_more = batchHasMore || (remainingLimit <= 0 && hasLaterDefinitionGroups); + yield { chunkData: yieldChunk, targetOp: targetOp }; + } + + if (batchHasMore || remainingLimit <= 0) { + return; + } + } +} + +export async function getDataBucketChangesV3( + ctx: MongoSyncBucketStorageContext, + options: GetCheckpointChangesOptions +): Promise> { + const limit = 1000; + const bucketStateUpdates = await ctx.db + .bucketStateV3(ctx.group_id) + .aggregate<{ _id: string; last_op: bigint }>( + [ + { + $match: { + last_op: { $gt: options.lastCheckpoint.checkpoint } + } + }, + { + $group: { + _id: '$_id.b', + last_op: { $max: '$last_op' } + } + }, + { + $sort: { + last_op: 1 + } + }, + { + $limit: limit + 1 + } + ], + { maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS } + ) + .toArray(); + + const buckets = bucketStateUpdates.map((doc) => doc._id); + const invalidateDataBuckets = buckets.length > limit; + + return { + invalidateDataBuckets, + updatedDataBuckets: invalidateDataBuckets ? new Set() : new Set(buckets) + }; +} + +export async function getParameterBucketChangesV3( + ctx: MongoSyncBucketStorageContext, + options: GetCheckpointChangesOptions +): Promise> { + const limit = 1000; + const indexIds = ctx.mapping.allParameterIndexIds(); + const collections = indexIds.map((indexId) => ({ + indexId, + collection: ctx.db.parameterIndexV3(ctx.group_id, indexId) + })); + if (collections.length == 0) { + return { + invalidateParameterBuckets: false, + updatedParameterLookups: new Set() + }; + } + const checkpointFilter = { + _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint } + }; + const pipelineForCollection = (indexId: string) => [ + { + $match: checkpointFilter + }, + { + $project: { + _id: 0, + lookup: 1, + indexId: { $literal: indexId } + } + } + ]; + const [firstCollection, ...remainingCollections] = collections; + const parameterUpdates = await firstCollection.collection + .aggregate<{ lookup: bson.Binary; indexId: string }>( + [ + ...pipelineForCollection(firstCollection.indexId), + ...remainingCollections.map((collection) => { + return { + $unionWith: { + coll: collection.collection.collectionName, + pipeline: pipelineForCollection(collection.indexId) + } + }; + }), + { + $limit: limit + 1 + } + ], + { + batchSize: limit + 2, + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + } + ) + .toArray(); + + const invalidateParameterUpdates = parameterUpdates.length > limit; + + return { + invalidateParameterBuckets: invalidateParameterUpdates, + updatedParameterLookups: invalidateParameterUpdates + ? new Set() + : new Set( + parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookupV3(p.lookup, p.indexId))) + ) + }; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts new file mode 100644 index 000000000..e3be5e8f0 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -0,0 +1,318 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { InternalOpId, storage } from '@powersync/service-core'; +import { BucketDataSource } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; +import { mongoTableId } from '../../../utils/util.js'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import { + BucketStateUpdate, + PersistedBatch, + SaveParameterDataOptions, + UpsertCurrentDataOptions +} from '../common/PersistedBatch.js'; +import { SourceTableKey } from '../models.js'; +import { + BucketParameterDocumentV3, + BucketStateDocumentV3, + CurrentDataDocumentV3, + serializeBucketDataV3, + SourceTableDocumentV3, + taggedBucketParameterDocumentToV3 +} from './models.js'; +import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; + +export class PersistedBatchV3 extends PersistedBatch { + declare protected readonly db: VersionedPowerSyncMongoV3; + + currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation }[] = []; + sourceTablePendingDeletes = new Map(); + + protected checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId { + if (definitionId == null) { + // This is required for V3 storage. + throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); + } + return definitionId; + } + + protected getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId { + return this.mapping.bucketSourceId(bucketSource); + } + + saveParameterData(data: SaveParameterDataOptions) { + const { sourceTable, sourceKey, evaluated } = data; + const remaining_lookups = new Map(); + + for (let lookup of data.existing_lookups) { + if (lookup.indexId == null) { + throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); + } + remaining_lookups.set(`${lookup.indexId}.${lookup.lookup.toString('base64')}`, lookup); + } + + for (let result of evaluated) { + const sourceDefinitionId = this.mapping.parameterLookupId(result.lookup.source); + const binLookup = serializeParameterLookupV3(result.lookup); + remaining_lookups.delete(`${sourceDefinitionId}.${binLookup.toString('base64')}`); + + const op_id = data.op_seq.next(); + this.debugLastOpId = op_id; + const values: BucketParameterDocumentV3 = { + _id: op_id, + key: { + t: mongoTableId(sourceTable.id), + k: sourceKey + } satisfies SourceTableKey, + lookup: binLookup, + bucket_parameters: result.bucketParameters + }; + this.bucketParameters.push({ + ...values, + index: sourceDefinitionId + }); + + this.currentSize += 200; + } + + for (let lookup of remaining_lookups.values()) { + const op_id = data.op_seq.next(); + this.debugLastOpId = op_id; + const indexId = lookup.indexId; + if (indexId == null) { + throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); + } + const values: BucketParameterDocumentV3 = { + _id: op_id, + key: { + t: mongoTableId(sourceTable.id), + k: sourceKey + } satisfies SourceTableKey, + lookup: lookup.lookup, + bucket_parameters: [] + }; + this.bucketParameters.push({ + ...values, + index: indexId + }); + + this.currentSize += 200; + } + } + + hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId) { + this.currentData.push({ + sourceTableId, + operation: { + deleteOne: { + filter: { _id: replicaId } + } + } + }); + this.currentSize += 50; + } + + softDeleteCurrentData( + sourceTableId: bson.ObjectId, + replicaId: storage.ReplicaId, + checkpointGreaterThan: InternalOpId + ) { + this.currentData.push({ + sourceTableId, + operation: { + updateOne: { + filter: { _id: replicaId }, + update: { + $set: { + data: null, + buckets: [] as CurrentDataDocumentV3['buckets'], + lookups: [] as CurrentDataDocumentV3['lookups'], + pending_delete: checkpointGreaterThan + } + }, + upsert: true + } + } + }); + const sourceTableKey = sourceTableId.toHexString(); + const existingPendingDelete = this.sourceTablePendingDeletes.get(sourceTableKey); + if (existingPendingDelete == null || checkpointGreaterThan > existingPendingDelete) { + this.sourceTablePendingDeletes.set(sourceTableKey, checkpointGreaterThan); + } + + this.currentSize += 50; + } + + upsertCurrentData(values: UpsertCurrentDataOptions) { + const buckets = values.buckets.map((bucket) => { + if (bucket.definitionId == null) { + throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); + } + return { + def: bucket.definitionId, + bucket: bucket.bucket, + table: bucket.table, + id: bucket.id + }; + }); + const lookups = values.lookups.map((lookup) => { + if (lookup.indexId == null) { + throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); + } + return { + i: lookup.indexId, + l: lookup.lookup + }; + }); + + this.currentData.push({ + sourceTableId: values.sourceTableId, + operation: { + updateOne: { + filter: { _id: values.replicaId }, + update: { + $set: { + data: values.data, + buckets, + lookups + }, + $unset: { pending_delete: 1 } + }, + upsert: true + } + } + }); + this.currentSize += (values.data?.length() ?? 0) + 100; + } + + protected get currentDataCount() { + return this.currentData.length; + } + + protected async flushBucketData(session: mongo.ClientSession) { + const operationsByDefinition = new Map(); + for (const document of this.bucketData) { + const existing = operationsByDefinition.get(document.bucketKey.definitionId) ?? []; + existing.push(document); + operationsByDefinition.set(document.bucketKey.definitionId, existing); + } + + for (const [definitionId, documents] of operationsByDefinition.entries()) { + await this.db.bucketDataV3(this.group_id, definitionId).bulkWrite( + documents.map((document) => ({ + insertOne: { + document: serializeBucketDataV3(document) + } + })), + { + session, + ordered: false + } + ); + } + } + + protected async flushBucketParameters(session: mongo.ClientSession) { + const operationsByIndex = new Map(); + for (const document of this.bucketParameters) { + const existing = operationsByIndex.get(document.index) ?? []; + existing.push(document); + operationsByIndex.set(document.index, existing); + } + + for (const [indexId, documents] of operationsByIndex.entries()) { + await this.db.parameterIndexV3(this.group_id, indexId).bulkWrite( + documents.map((document) => ({ + insertOne: { + document: taggedBucketParameterDocumentToV3(document) + } + })), + { + session, + ordered: false + } + ); + } + } + + protected async flushCurrentData(session: mongo.ClientSession) { + const operationsBySourceTable = new Map(); + for (const operation of this.currentData) { + const sourceTableId = operation.sourceTableId.toHexString(); + const existing = operationsBySourceTable.get(sourceTableId) ?? []; + existing.push(operation); + operationsBySourceTable.set(sourceTableId, existing); + } + + const sourceTableUpdates: mongo.AnyBulkWriteOperation[] = [ + ...this.sourceTablePendingDeletes.entries() + ].map(([key, value]) => { + return { + updateOne: { + filter: { _id: new bson.ObjectId(key) }, + update: { + $max: { + latest_pending_delete: value + } + } + } + }; + }); + + if (sourceTableUpdates.length > 0) { + await this.db.sourceTablesV3(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false }); + } + + for (const operations of operationsBySourceTable.values()) { + const sourceTableId = operations[0]!.sourceTableId; + await this.db.sourceRecordsV3(this.group_id, sourceTableId).bulkWrite( + operations.map((entry) => entry.operation), + { + session, + ordered: true + } + ); + } + } + + protected async flushBucketStates(session: mongo.ClientSession) { + await this.db.bucketStateV3(this.group_id).bulkWrite(this.getBucketStateUpdates(), { + session, + ordered: false + }); + } + + protected resetCurrentData() { + this.currentData = []; + this.sourceTablePendingDeletes.clear(); + } + + private getBucketStateUpdates(): mongo.AnyBulkWriteOperation[] { + return Array.from(this.bucketStates.values()).map((state: BucketStateUpdate) => { + if (state.definitionId == null) { + throw new ReplicationAssertionError('Expected bucket definition id when incrementalReprocessing is enabled'); + } + return { + updateOne: { + filter: { + _id: { + d: state.definitionId, + b: state.bucket + } + }, + update: { + $set: { + last_op: state.lastOp + }, + $inc: { + 'estimate_since_compact.count': state.incrementCount, + 'estimate_since_compact.bytes': state.incrementBytes + } + }, + upsert: true + } + } satisfies mongo.AnyBulkWriteOperation; + }); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/SingleBucketStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/SingleBucketStoreV3.ts new file mode 100644 index 000000000..037773723 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/SingleBucketStoreV3.ts @@ -0,0 +1,68 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { InternalOpId } from '@powersync/service-core'; +import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js'; +import { + BucketDataDocumentGeneric, + BucketDataDocumentGenericId, + SingleBucketStore +} from '../common/SingleBucketStore.js'; +import { BucketDataProperties } from '../models.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; +import { BucketDataDocumentV3, BucketDataKeyV3, loadBucketDataDocumentV3, serializeBucketDataV3 } from './models.js'; + +export class SingleBucketStoreV3 implements SingleBucketStore { + public readonly collection: mongo.Collection; + + constructor( + private db: VersionedPowerSyncMongoV3, + public readonly key: BucketKey + ) { + this.collection = db.bucketDataV3( + key.replicationStreamId, + key.definitionId + ) as unknown as mongo.Collection; + } + + docId(o: InternalOpId): BucketDataDocumentGenericId { + // `satisfies BucketDataKeyV3` checks that we use the correct type for V3 storage + // `as BucketDataDocumentGenericId` does a cast to get the interface virtual type + return { + b: this.key.bucket, + o + } satisfies BucketDataKeyV3 as BucketDataDocumentGenericId; + } + + get minId(): BucketDataDocumentGenericId { + return { + b: this.key.bucket, + o: new mongo.MinKey() + } as any; // No way to properly type this + } + + get maxId(): BucketDataDocumentGenericId { + return { + b: this.key.bucket, + o: new mongo.MaxKey() + } as any; // No way to properly type this + } + + toPersistedDocument(source: Omit): BucketDataDocumentGeneric { + return serializeBucketDataV3({ bucketKey: this.key, ...source }) as BucketDataDocumentGeneric; + } + + fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc { + return loadBucketDataDocumentV3(this.key, doc as BucketDataDocumentV3); + } + + fromPartialPersistedDocument( + doc: Pick + ): Pick { + const document = doc as Pick; + const { _id, ...rest } = document; + return { + bucketKey: this.key, + o: _id.o, + ...rest + } as Pick; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts new file mode 100644 index 000000000..aae823f9d --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts @@ -0,0 +1,226 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; +import { Logger } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; +import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js'; +import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; +import { cacheKey } from '../OperationBatch.js'; +import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js'; +import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; +import { CurrentDataDocumentV3, SourceTableDocumentV3 } from './models.js'; + +export class SourceRecordStoreV3 implements SourceRecordStore { + constructor( + private readonly db: VersionedPowerSyncMongoV3, + private readonly groupId: number, + private readonly mapping: BucketDefinitionMapping + ) {} + + mapEvaluatedBuckets(evaluated: EvaluatedRow[]): LoadedSourceRecord['buckets'] { + return evaluated.map((entry) => ({ + definitionId: this.mapping.bucketSourceId(entry.source), + bucket: entry.bucket, + table: entry.table, + id: entry.id + })); + } + + mapParameterLookups(paramEvaluated: EvaluatedParameters[]): LoadedSourceRecord['lookups'] { + return paramEvaluated.map((entry) => ({ + indexId: this.mapping.parameterLookupId(entry.lookup.source), + lookup: serializeParameterLookupV3(entry.lookup) + })); + } + + private createLoadedDocument( + sourceTableId: bson.ObjectId, + id: storage.ReplicaId, + data: bson.Binary | null, + buckets: CurrentDataDocumentV3['buckets'], + lookups: CurrentDataDocumentV3['lookups'] + ): LoadedSourceRecord { + return { + sourceTableId, + replicaId: id, + data, + buckets: buckets.map((bucket) => ({ + definitionId: bucket.def, + bucket: bucket.bucket, + table: bucket.table, + id: bucket.id + })), + lookups: lookups.map((lookup) => ({ + indexId: lookup.i, + lookup: lookup.l + })), + cacheKey: cacheKey(sourceTableId, id) + }; + } + + async loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise> { + const sizes = new Map(); + for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { + const filter = { + _id: { $in: replicaIds as any[] } + } as unknown as mongo.Filter; + const sizeCursor: mongo.AggregationCursor = this.db + .sourceRecordsV3(this.groupId, sourceTableId) + .aggregate( + [ + { + $match: filter + }, + { + $project: { + _id: 1, + size: { $bsonSize: '$$ROOT' } + } + } + ], + { session } + ); + for await (const doc of sizeCursor.stream()) { + sizes.set(cacheKey(sourceTableId, doc._id), doc.size); + } + } + return sizes; + } + + async loadDocuments( + session: mongo.ClientSession, + entries: SourceRecordLookupEntry[], + idsOnly: boolean + ): Promise> { + const documents = new Map(); + const projection = idsOnly ? { _id: 1 } : undefined; + for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { + const filter = { + _id: { $in: replicaIds as any[] } + } as unknown as mongo.Filter; + const cursor = this.db.sourceRecordsV3(this.groupId, sourceTableId).find(filter, { session, projection }); + for await (const doc of cursor.stream()) { + const loaded = this.createLoadedDocument( + sourceTableId, + doc._id, + idsOnly ? null : doc.data, + idsOnly ? [] : doc.buckets, + idsOnly ? [] : doc.lookups + ); + documents.set(loaded.cacheKey, loaded); + } + } + return documents; + } + + async loadTruncateBatch( + session: mongo.ClientSession, + sourceTableId: bson.ObjectId, + limit: number + ): Promise { + const cursor = this.db.sourceRecordsV3(this.groupId, sourceTableId).find( + { + pending_delete: { $exists: false } + }, + { + projection: { + _id: 1, + buckets: 1, + lookups: 1 + }, + limit, + session + } + ); + return (await cursor.toArray()).map((doc) => + this.createLoadedDocument(sourceTableId, doc._id, null, doc.buckets, doc.lookups) + ); + } + + async postCommitCleanup(lastCheckpoint: bigint, logger: Logger): Promise { + // This cleans up soft deletes in source_records collections. + // Since there may be a lot (100+) of these collections in some cases, we track which + // ones have dirty deletes in source_tables. + + const dirtySourceTables = await this.db + .sourceTablesV3(this.groupId) + .find( + { + latest_pending_delete: { $exists: true } + }, + { + projection: { _id: 1, latest_pending_delete: 1 } + } + ) + .toArray(); + + let deletedCount = 0; + const sourceTableUpdates: mongo.AnyBulkWriteOperation[] = []; + for (const sourceTable of dirtySourceTables) { + const collection = this.db.sourceRecordsV3(this.groupId, sourceTable._id); + const result = await this.deletePendingDeletes(collection, sourceTable._id, lastCheckpoint, logger); + deletedCount += result.deletedCount; + + if (sourceTable.latest_pending_delete != null && sourceTable.latest_pending_delete <= lastCheckpoint) { + sourceTableUpdates.push({ + updateOne: { + filter: { + _id: sourceTable._id, + // If the source table received more writes in the meantime, this will filter it out + latest_pending_delete: sourceTable.latest_pending_delete + }, + update: { + $unset: { + latest_pending_delete: 1 + } + } + } + }); + } + } + + if (sourceTableUpdates.length > 0) { + await this.db.sourceTablesV3(this.groupId).bulkWrite(sourceTableUpdates, { ordered: false }); + } + if (deletedCount > 0) { + logger.info(`Cleaned up ${deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}`); + } + } + + private async deletePendingDeletes( + collection: mongo.Collection, + sourceTableId: bson.ObjectId, + lastCheckpoint: bigint, + logger: Logger + ) { + return retryOnMongoMaxTimeMSExpired( + () => + collection.deleteMany( + { + pending_delete: { $exists: true, $lte: lastCheckpoint } + }, + { + maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS + } + ), + { + retryDelayMs: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS / 5, + onRetry: (n: number) => { + logger.warn(`Cleared batch ${n} of pending deletes for source table ${sourceTableId}, continuing...`); + } + } + ); + } + + private groupEntries(entries: SourceRecordLookupEntry[]): Map { + const grouped = new Map(); + for (const entry of entries) { + const existing = grouped.get(entry.sourceTableId) ?? []; + existing.push(entry.replicaId); + grouped.set(entry.sourceTableId, existing); + } + return grouped; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts new file mode 100644 index 000000000..be1da41ac --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts @@ -0,0 +1,112 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; +import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js'; +import { CommonSourceTableDocument } from '../models.js'; +import { + BucketDataDocumentV3, + BucketParameterDocumentV3, + BucketStateDocumentV3, + CurrentDataDocumentV3, + SourceTableDocumentV3 +} from './models.js'; + +export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { + sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection { + const collectionName = this.sourceRecordsCollectionName(replicationStreamId, sourceTableId); + return this.db.collection(collectionName); + } + + async listSourceRecordCollectionsV3(replicationStreamId: number): Promise[]> { + return this.listCollectionsByPrefix(`source_records_${replicationStreamId}_`); + } + + async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) { + await this.sourceRecordsV3(replicationStreamId, sourceTableId).createIndex( + { + pending_delete: 1 + }, + { + partialFilterExpression: { pending_delete: { $exists: true } }, + name: 'pending_delete' + } + ); + } + + commonSourceTables(replicationStreamId: number): mongo.Collection { + return this.sourceTablesV3(replicationStreamId) as mongo.Collection; + } + + bucketStateV3(replicationStreamId: number): mongo.Collection { + return this.db.collection(`bucket_state_${replicationStreamId}`); + } + + parameterIndexV3( + replicationStreamId: number, + indexId: ParameterIndexId + ): mongo.Collection { + return this.db.collection(`parameter_index_${replicationStreamId}_${indexId}`); + } + + sourceTablesV3(replicationStreamId: number): mongo.Collection { + return this.db.collection(this.sourceTableCollectionName(replicationStreamId)); + } + + async initializeStreamStorage(replicationStreamId: number) { + const sourceTables = this.sourceTablesV3(replicationStreamId); + const bucketState = this.bucketStateV3(replicationStreamId); + await sourceTables.createIndex( + { + connection_id: 1, + schema_name: 1, + table_name: 1, + relation_id: 1 + }, + { + name: 'source_lookup' + } + ); + await sourceTables.createIndex( + { + latest_pending_delete: 1 + }, + { + partialFilterExpression: { latest_pending_delete: { $exists: true } }, + name: 'latest_pending_delete' + } + ); + await bucketState.createIndex( + { + last_op: 1 + }, + { name: 'bucket_updates', unique: true } + ); + await bucketState.createIndex( + { + 'estimate_since_compact.count': -1 + }, + { name: 'dirty_count' } + ); + } + + bucketDataV3(replicationStreamId: number, definitionId: BucketDefinitionId) { + return this.db.collection(`bucket_data_${replicationStreamId}_${definitionId}`); + } + + listBucketDataCollectionsV3(replicationStreamId: number) { + return this.upstream.listBucketDataCollectionsV3(replicationStreamId); + } + + async listParameterIndexCollectionsV3( + replicationStreamId: number + ): Promise<{ collection: mongo.Collection; indexId: ParameterIndexId }[]> { + const prefix = `parameter_index_${replicationStreamId}_`; + const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => ({ + collection: this.db.collection(collection.name), + indexId: collection.name.slice(prefix.length) + })); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts new file mode 100644 index 000000000..64f0c5516 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts @@ -0,0 +1,96 @@ +import { InternalOpId } from '@powersync/service-core'; +import * as bson from 'bson'; +import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; +import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js'; +import { + BucketDataDocumentBase, + BucketDataKey, + BucketParameterDocumentBase, + BucketStateDocumentBase, + CurrentBucket, + ReplicaId, + SourceTableDocument, + SourceTableKey, + TaggedBucketParameterDocument +} from '../models.js'; + +export interface CurrentBucketV3 extends CurrentBucket { + def: BucketDefinitionId; +} + +export interface RecordedLookupV3 { + i: ParameterIndexId; + l: bson.Binary; +} + +export interface CurrentDataDocumentV3 { + _id: ReplicaId; + data: bson.Binary | null; + buckets: CurrentBucketV3[]; + lookups: RecordedLookupV3[]; + /** + * If set, this can be deleted, once there is a consistent checkpoint >= pending_delete. + * + * This must only be set if buckets = [], lookups = []. + */ + pending_delete?: bigint; +} + +export interface BucketParameterDocumentV3 extends BucketParameterDocumentBase {} + +export type BucketDataKeyV3 = BucketDataKey; + +export interface BucketDataDocumentV3 extends BucketDataDocumentBase { + _id: BucketDataKeyV3; +} + +export function serializeBucketDataV3(document: BucketDataDoc): BucketDataDocumentV3 { + const { bucketKey, o } = document; + return { + _id: { + b: bucketKey.bucket, + o: o + }, + // List fields directly, so that we don't accidentally persist any unknown fields + op: document.op, + source_table: document.source_table, + source_key: document.source_key, + table: document.table, + row_id: document.row_id, + checksum: document.checksum, + data: document.data, + target_op: document.target_op + }; +} + +export function loadBucketDataDocumentV3( + context: Pick, + doc: BucketDataDocumentV3 +): BucketDataDoc { + const { _id, ...rest } = doc; + return { + bucketKey: { + ...context, + bucket: _id.b + }, + o: _id.o, + ...rest + }; +} + +export function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3 { + const { index: _index, ...rest } = document; + return rest as BucketParameterDocumentV3; +} + +export interface SourceTableDocumentV3 extends SourceTableDocument { + bucket_data_source_ids: BucketDefinitionId[]; + parameter_lookup_source_ids: ParameterIndexId[]; + latest_pending_delete?: InternalOpId | undefined; +} + +export interface BucketStateDocumentV3 extends BucketStateDocumentBase { + _id: BucketStateDocumentBase['_id'] & { + d: BucketDefinitionId; + }; +} diff --git a/modules/module-mongodb-storage/src/storage/storage-index.ts b/modules/module-mongodb-storage/src/storage/storage-index.ts index a534c3aca..bcc83ab1b 100644 --- a/modules/module-mongodb-storage/src/storage/storage-index.ts +++ b/modules/module-mongodb-storage/src/storage/storage-index.ts @@ -1,14 +1,17 @@ export * as test_utils from '../utils/test-utils.js'; export * from '../utils/util.js'; +export * from './implementation/BucketDefinitionMapping.js'; +export * from './implementation/common/PersistedBatch.js'; +export * from './implementation/createMongoSyncBucketStorage.js'; export * from './implementation/db.js'; export * from './implementation/models.js'; -export * from './implementation/MongoBucketBatch.js'; export * from './implementation/MongoIdSequence.js'; +export * from './implementation/MongoPersistedSyncRules.js'; export * from './implementation/MongoPersistedSyncRulesContent.js'; export * from './implementation/MongoStorageProvider.js'; -export * from './implementation/MongoSyncBucketStorage.js'; export * from './implementation/MongoSyncRulesLock.js'; export * from './implementation/OperationBatch.js'; -export * from './implementation/PersistedBatch.js'; +export * from './implementation/v1/models.js'; +export * from './implementation/v3/models.js'; export * from './MongoBucketStorage.js'; export * from './MongoReportStorage.js'; diff --git a/modules/module-mongodb-storage/src/utils/util.ts b/modules/module-mongodb-storage/src/utils/util.ts index 59c2f451d..88f55c6fc 100644 --- a/modules/module-mongodb-storage/src/utils/util.ts +++ b/modules/module-mongodb-storage/src/utils/util.ts @@ -1,10 +1,12 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; -import { ServiceAssertionError } from '@powersync/lib-services-framework'; +import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; import * as bson from 'bson'; import * as crypto from 'crypto'; +import * as timers from 'node:timers/promises'; import * as uuid from 'uuid'; -import { BucketDataDocument } from '../storage/implementation/models.js'; +import { BucketDataDoc } from '../storage/implementation/common/BucketDataDoc.js'; export function idPrefixFilter(prefix: Partial, rest: (keyof T)[]): mongo.Condition { let filter = { @@ -69,10 +71,10 @@ export async function readSingleBatch(cursor: mongo.AbstractCursor): Promi } } -export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry { +export function mapOpEntry(row: BucketDataDoc): utils.OplogEntry { if (row.op == 'PUT' || row.op == 'REMOVE') { return { - op_id: utils.internalToExternalOpId(row._id.o), + op_id: utils.internalToExternalOpId(row.o), op: row.op, object_type: row.table, object_id: row.row_id, @@ -84,7 +86,7 @@ export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry { // MOVE, CLEAR return { - op_id: utils.internalToExternalOpId(row._id.o), + op_id: utils.internalToExternalOpId(row.o), op: row.op, checksum: Number(row.checksum) }; @@ -129,6 +131,33 @@ export function setSessionSnapshotTime(session: mongo.ClientSession, time: bson. } } +export async function retryOnMongoMaxTimeMSExpired( + operation: () => Promise, + options: { + signal?: AbortSignal; + abortMessage?: string; + retryDelayMs: number; + onRetry?: (retryCount: number) => void; + } +): Promise { + let retryCount = 0; + while (true) { + if (options.signal?.aborted) { + throw new ReplicationAbortedError(options.abortMessage ?? 'Aborted MongoDB operation', options.signal.reason); + } + try { + return await operation(); + } catch (e) { + if (!lib_mongo.isMongoServerError(e) || e.codeName !== 'MaxTimeMSExpired') { + throw e; + } + retryCount += 1; + options.onRetry?.(retryCount); + await timers.setTimeout(options.retryDelayMs); + } + } +} + export const createPaginatedConnectionQuery = async ( query: mongo.Filter, collection: mongo.Collection, diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index e277f6413..003cbab59 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -1,7 +1,7 @@ +import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; import { storage, SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; -import { MongoCompactor } from '../../src/storage/implementation/MongoCompactor.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; describe('Mongo Sync Bucket Storage Compact', () => { @@ -64,9 +64,16 @@ bucket_definitions: test('full compact', async () => { const { bucketStorage, checkpoint, factory, syncRules } = await setup(); + const storageDb = bucketStorage.db; // Simulate bucket_state from old version not being available - await factory.db.bucket_state.deleteMany({}); + if (storageDb.storageConfig.incrementalReprocessing) { + // This should actually never happen on V3, but we test this anyway. + // Can remove this if it causes issues in the future. + await (storageDb as VersionedPowerSyncMongoV3).bucketStateV3(bucketStorage.group_id).deleteMany({}); + } else { + await factory.db.bucket_state.deleteMany({}); + } await bucketStorage.compact({ clearBatchLimit: 200, @@ -108,6 +115,7 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const storageDb = (bucketStorage as any).db; await populate(bucketStorage, 2); const { checkpoint } = await bucketStorage.getCheckpoint(); @@ -158,35 +166,54 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const storageDb = bucketStorage.db; // This simulates bucket_state created using bigint bytes. // This typically happens when buckets get very large (> 2GiB). We don't want to create that much // data in the tests, so we directly insert the bucket_state here. - await factory.db.bucket_state.insertOne({ - _id: { - g: bucketStorage.group_id, - b: 'global[]' - }, - last_op: 5n, - compacted_state: { - op_id: 3n, - count: 3, - checksum: 0n, - bytes: 7n - }, - estimate_since_compact: { - count: 2, - bytes: 5n - } - }); - - // This test uses a couple of internal APIs of the compactor - there is no simple way - // to test this using the current public APIs. - const compactor = new MongoCompactor(bucketStorage, (bucketStorage as any).db, { - maxOpId: 5n - }); - - const dirtyBuckets = (compactor as any).dirtyBucketBatches({ + if (storageDb.storageConfig.incrementalReprocessing) { + const bucketStateCollection = (storageDb as VersionedPowerSyncMongoV3).bucketStateV3(bucketStorage.group_id); + await bucketStateCollection.insertOne({ + _id: { + d: '1', + b: 'global[]' + }, + last_op: 5n, + compacted_state: { + op_id: 3n, + count: 3, + checksum: 0n, + bytes: 7n + }, + estimate_since_compact: { + count: 2, + bytes: 5n + } + }); + } else { + await factory.db.bucket_state.insertOne({ + _id: { + g: bucketStorage.group_id, + b: 'global[]' + }, + last_op: 5n, + compacted_state: { + op_id: 3n, + count: 3, + checksum: 0n, + bytes: 7n + }, + estimate_since_compact: { + count: 2, + bytes: 5n + } + }); + } + + // This test uses a couple of "internal" APIs of the compactor. + const compactor = bucketStorage.createMongoCompactor({ maxOpId: 5n }); + + const dirtyBuckets = compactor.dirtyBucketBatches({ minBucketChanges: 1, minChangeRatio: 0.39 }); @@ -205,6 +232,7 @@ bucket_definitions: expect(checksumBuckets).toEqual([ { bucket: 'global[]', + definitionId: storageDb.storageConfig.incrementalReprocessing ? '1' : null, estimatedCount: 5 } ]); diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 7e53340c8..6afa1c38b 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -1,6 +1,14 @@ -import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; +import { RequestParameters } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; +import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; +import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; +import { SyncRuleDocument } from '../../src/storage/implementation/models.js'; +import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js'; +import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; +import { CurrentBucketV3 } from '../../src/storage/implementation/v3/models.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) { @@ -126,12 +134,350 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor // Test that the checksum type is correct. // Specifically, test that it never persisted as double. - const mongoFactory = factory as any; - const checksumTypes = await mongoFactory.db.bucket_data - .aggregate([{ $group: { _id: { $type: '$checksum' }, count: { $sum: 1 } } }]) - .toArray(); + const mongoFactory = factory as MongoBucketStorage; + const checksumTypes = + storageVersion >= 3 + ? ( + await Promise.all( + ( + await mongoFactory.db.db + .listCollections({ name: new RegExp(`^bucket_data_${syncRules.id}_`) }, { nameOnly: true }) + .toArray() + ).map((collection: { name: string }) => + mongoFactory.db.db + .collection(collection.name) + .aggregate([{ $group: { _id: { $type: '$checksum' }, count: { $sum: 1 } } }]) + .toArray() + ) + ) + ).flat() + : await mongoFactory.db.bucket_data + .aggregate([{ $group: { _id: { $type: '$checksum' }, count: { $sum: 1 } } }]) + .toArray(); expect(checksumTypes).toEqual([{ _id: 'long', count: 4 }]); }); + + test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => { + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` + bucket_definitions: + global: + parameters: + - SELECT owner_id FROM test WHERE id = token_parameters.test + data: + - SELECT id, description, owner_id FROM test WHERE id = bucket.owner_id + `, + { storageVersion } + ) + ); + const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'shape-check', + description: 'shape', + owner_id: 'user-1' + }, + afterReplicaId: test_utils.rid('shape-check') + }); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); + + const checkpoint = await bucketStorage.getCheckpoint(); + const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { test: 'shape-check' } }), {}); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; + const buckets = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => l.indexKey)).toEqual([['shape-check']]); + expect(lookups[0].indexId).toEqual('1'); + + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets).toEqual([{ owner_id: 'user-1' }]); + return parameter_sets; + } + }); + expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'global["user-1"]').bucket]); + + const mongoFactory = factory as MongoBucketStorage; + const db = (bucketStorage as MongoSyncBucketStorage).db as VersionedPowerSyncMongoV3; + const currentDataCollections = await db.listSourceRecordCollectionsV3(syncRules.id); + const currentData = await currentDataCollections[0]?.findOne({}); + const firstBucket: CurrentBucketV3 | undefined = currentData?.buckets[0] as CurrentBucketV3 | undefined; + expect(firstBucket?.def).toMatch(/^[0-9a-f]+$/); + + const bucketCollections = await mongoFactory.db.db + .listCollections({ name: new RegExp(`^bucket_data_${syncRules.id}_`) }, { nameOnly: true }) + .toArray(); + expect( + bucketCollections.some((collection) => collection.name === `bucket_data_${syncRules.id}_${firstBucket?.def}`) + ).toBe(true); + + const syncRule = await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id }); + const ruleMapping: SyncRuleDocument['rule_mapping'] | undefined = syncRule?.rule_mapping; + expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0); + + const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0] as string | undefined; + expect(parameterIndexId).toBeDefined(); + const parameterEntry = await db.parameterIndexV3(syncRules.id, parameterIndexId!).findOne({}); + expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']); + }); + + test.runIf(storageVersion < 3)('uses a single current_data collection for v1 source records', async () => { + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` + bucket_definitions: + global: + data: + - SELECT id, description FROM test + `, + { storageVersion } + ) + ); + const bucketStorage = factory.getInstance(syncRules); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'shape-check', + description: 'shape' + }, + afterReplicaId: test_utils.rid('shape-check') + }); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); + + const mongoFactory = factory as MongoBucketStorage; + expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1); + + const sourceRecordCollections = await mongoFactory.db.db + .listCollections({ name: new RegExp(`^source_records_${syncRules.id}_`) }, { nameOnly: true }) + .toArray(); + expect(sourceRecordCollections).toEqual([]); + }); + + test.runIf(storageVersion < 3)('clear removes v1 current_data rows', async () => { + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` + bucket_definitions: + global: + data: + - SELECT id, description FROM test + `, + { storageVersion } + ) + ); + const bucketStorage = factory.getInstance(syncRules); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'clear-check', + description: 'shape' + }, + afterReplicaId: test_utils.rid('clear-check') + }); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); + + const mongoFactory = factory as MongoBucketStorage; + expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1); + + await bucketStorage.clear(); + + expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(0); + }); + + test.runIf(storageVersion < 3)('storage metrics include v1 current_data', async () => { + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` + bucket_definitions: + global: + data: + - SELECT id, description FROM test + `, + { storageVersion } + ) + ); + const bucketStorage = factory.getInstance(syncRules); + const metricsBefore = await factory.getStorageMetrics(); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'metric-check', + description: 'shape' + }, + afterReplicaId: test_utils.rid('metric-check') + }); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); + + const mongoFactory = factory as MongoBucketStorage; + expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1); + + const metricsAfter = await factory.getStorageMetrics(); + expect(metricsAfter.replication_size_bytes).toBeGreaterThan(metricsBefore.replication_size_bytes); + }); + + test.runIf(storageVersion >= 3)( + 'loads parameter checkpoint changes across all v3 parameter index collections', + async () => { + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` + bucket_definitions: + by_owner: + parameters: + - SELECT owner_id FROM test WHERE id = token_parameters.owner_lookup + data: + - SELECT id, owner_id FROM test WHERE owner_id = bucket.owner_id + by_category: + parameters: + - SELECT category_id FROM test WHERE id = token_parameters.category_lookup + data: + - SELECT id, category_id FROM test WHERE category_id = bucket.category_id + `, + { storageVersion } + ) + ); + const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const previousCheckpoint = await bucketStorage.getCheckpoint(); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'shape-check', + owner_id: 'user-1', + category_id: 'cat-1' + }, + afterReplicaId: test_utils.rid('shape-check') + }); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); + + const nextCheckpoint = await bucketStorage.getCheckpoint(); + const changes = await bucketStorage.getCheckpointChanges({ + lastCheckpoint: previousCheckpoint, + nextCheckpoint + }); + + expect(changes.invalidateParameterBuckets).toBe(false); + expect(changes.updatedParameterLookups).toEqual(new Set(['["1","","shape-check"]', '["2","","shape-check"]'])); + } + ); + + test.runIf(storageVersion >= 3)('cleans pending deletes only for tracked v3 source tables', async () => { + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` + bucket_definitions: + global: + data: + - SELECT id, description FROM test + `, + { storageVersion } + ) + ); + + const mongoFactory = factory as MongoBucketStorage; + const bucketStorage = mongoFactory.getInstance(syncRules) as any; + const db = bucketStorage.db; + await db.initializeStreamStorage(syncRules.id); + + const sourceTableA = new bson.ObjectId(); + const sourceTableB = new bson.ObjectId(); + await db.sourceTablesV3(syncRules.id).insertMany([ + { + _id: sourceTableA, + connection_id: 1, + relation_id: 'a', + schema_name: 'public', + table_name: 'table_a', + replica_id_columns: null, + replica_id_columns2: [], + snapshot_done: true, + snapshot_status: undefined, + bucket_data_source_ids: [], + parameter_lookup_source_ids: [], + latest_pending_delete: 9n + }, + { + _id: sourceTableB, + connection_id: 1, + relation_id: 'b', + schema_name: 'public', + table_name: 'table_b', + replica_id_columns: null, + replica_id_columns2: [], + snapshot_done: true, + snapshot_status: undefined, + bucket_data_source_ids: [], + parameter_lookup_source_ids: [], + latest_pending_delete: 12n + } + ]); + + await db.sourceRecordsV3(syncRules.id, sourceTableA).insertMany([ + { _id: 'deleted-1', data: null, buckets: [], lookups: [], pending_delete: 5n }, + { _id: 'deleted-2', data: null, buckets: [], lookups: [], pending_delete: 9n }, + { _id: 'active', data: null, buckets: [], lookups: [] } + ]); + await db + .sourceRecordsV3(syncRules.id, sourceTableB) + .insertMany([{ _id: 'later-delete', data: null, buckets: [], lookups: [], pending_delete: 12n }]); + + const store = new SourceRecordStoreV3(db, syncRules.id, bucketStorage.sync_rules.mapping); + const logger = { info() {} } as any; + + await store.postCommitCleanup(6n, logger); + + expect(await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: 5n })).toBe(0); + expect(await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: 9n })).toBe(1); + expect(await db.sourceRecordsV3(syncRules.id, sourceTableB).countDocuments({ pending_delete: 12n })).toBe(1); + expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableA }))?.latest_pending_delete).toBe(9n); + expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableB }))?.latest_pending_delete).toBe(12n); + + await store.postCommitCleanup(10n, logger); + + expect( + await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: { $exists: true } }) + ).toBe(0); + expect( + (await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableA }))?.latest_pending_delete + ).toBeUndefined(); + expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableB }))?.latest_pending_delete).toBe(12n); + }); } describe('sync - mongodb', () => { diff --git a/modules/module-postgres-storage/src/storage/current-data-table.ts b/modules/module-postgres-storage/src/storage/current-data-table.ts deleted file mode 100644 index 27f62ef14..000000000 --- a/modules/module-postgres-storage/src/storage/current-data-table.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { ServiceAssertionError } from '@powersync/lib-services-framework'; -import { storage } from '@powersync/service-core'; - -export const V1_CURRENT_DATA_TABLE = 'current_data'; -export const V3_CURRENT_DATA_TABLE = 'v3_current_data'; - -/** - * The table used by a specific storage version for general current_data access. - */ -export function getCommonCurrentDataTable(storageConfig: storage.StorageVersionConfig) { - return storageConfig.softDeleteCurrentData ? V3_CURRENT_DATA_TABLE : V1_CURRENT_DATA_TABLE; -} - -export function getV1CurrentDataTable(storageConfig: storage.StorageVersionConfig) { - if (storageConfig.softDeleteCurrentData) { - throw new ServiceAssertionError('current_data table cannot be used when softDeleteCurrentData is enabled'); - } - return V1_CURRENT_DATA_TABLE; -} - -export function getV3CurrentDataTable(storageConfig: storage.StorageVersionConfig) { - if (!storageConfig.softDeleteCurrentData) { - throw new ServiceAssertionError('v3_current_data table cannot be used when softDeleteCurrentData is disabled'); - } - return V3_CURRENT_DATA_TABLE; -} diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index 77ce26cf2..315fb4bfa 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -23,7 +23,7 @@ import { reduceBucket, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; +import { METRICS_HELPER, StorageDataHelpers, test_utils } from '@powersync/service-core-tests'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; import * as postgres_storage from '@powersync/service-module-postgres-storage'; import * as timers from 'node:timers/promises'; @@ -97,6 +97,7 @@ bucket_definitions: `; const syncRules = await f.updateSyncRules(updateSyncRulesFromYaml(syncRuleContent, { storageVersion })); const storage = f.getInstance(syncRules); + const helpers = new StorageDataHelpers(storage, syncRules); abortController = new AbortController(); const options: WalStreamOptions = { abort_signal: abortController.signal, @@ -182,50 +183,11 @@ bucket_definitions: } const checkpoint = (await storage.getCheckpoint()).checkpoint; - if (f instanceof mongo_storage.storage.MongoBucketStorage) { - const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) - .filter((row) => row._id.o <= checkpoint) - .map(mongo_storage.storage.mapOpEntry); - await storage.compact({ maxOpId: checkpoint }); - const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) - .filter((row) => row._id.o <= checkpoint) - .map(mongo_storage.storage.mapOpEntry); - - test_utils.validateCompactedBucket(opsBefore, opsAfter); - } else if (f instanceof postgres_storage.PostgresBucketStorageFactory) { - const { db } = f; - const opsBefore = ( - await db.sql` - SELECT - * - FROM - bucket_data - WHERE - op_id <= ${{ type: 'int8', value: checkpoint }} - ORDER BY - op_id ASC - ` - .decoded(postgres_storage.models.BucketData) - .rows() - ).map(postgres_storage.utils.mapOpEntry); - await storage.compact({ maxOpId: checkpoint }); - const opsAfter = ( - await db.sql` - SELECT - * - FROM - bucket_data - WHERE - op_id <= ${{ type: 'int8', value: checkpoint }} - ORDER BY - op_id ASC - ` - .decoded(postgres_storage.models.BucketData) - .rows() - ).map(postgres_storage.utils.mapOpEntry); - - test_utils.validateCompactedBucket(opsBefore, opsAfter); - } + const opsBefore = await helpers.getBucketData('global[]', checkpoint); + await storage.compact({ maxOpId: checkpoint }); + const opsAfter = await helpers.getBucketData('global[]', checkpoint); + + test_utils.validateCompactedBucket(opsBefore, opsAfter); } }; @@ -247,20 +209,6 @@ bucket_definitions: return bson.deserialize(doc.data.buffer) as SqliteRow; }); expect(transformed).toEqual([]); - - // Check that each PUT has a REMOVE - const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray(); - - // All a single bucket in this test - const bucket = ops.map((op) => mongo_storage.storage.mapOpEntry(op)); - const reduced = test_utils.reduceBucket(bucket); - expect(reduced).toMatchObject([ - { - op_id: '0', - op: 'CLEAR' - } - // Should contain no additional data - ]); } else if (f instanceof postgres_storage.storage.PostgresBucketStorageFactory) { const { db } = f; // Check that all inserts have been deleted again @@ -301,6 +249,19 @@ bucket_definitions: // Should contain no additional data ]); } + + // Check that each PUT has a REMOVE + const checkpoint = (await storage.getCheckpoint()).checkpoint; + const ops = await helpers.getBucketData('global[]', checkpoint); + + const reduced = test_utils.reduceBucket(ops); + expect(reduced).toMatchObject([ + { + op_id: '0', + op: 'CLEAR' + } + // Should contain no additional data + ]); } abortController.abort(); diff --git a/modules/module-postgres/test/src/wal_stream_utils.ts b/modules/module-postgres/test/src/wal_stream_utils.ts index 22429269a..1406e33b2 100644 --- a/modules/module-postgres/test/src/wal_stream_utils.ts +++ b/modules/module-postgres/test/src/wal_stream_utils.ts @@ -7,22 +7,19 @@ import { initializeCoreReplicationMetrics, InternalOpId, LEGACY_STORAGE_VERSION, - OplogEntry, settledPromise, storage, - STORAGE_VERSION_CONFIG, SyncRulesBucketStorage, unsettledPromise, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, METRICS_HELPER, StorageDataHelpers, test_utils } from '@powersync/service-core-tests'; import * as pgwire from '@powersync/service-jpgwire'; import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js'; export class WalStreamTestContext implements AsyncDisposable { private _walStream?: WalStream; private abortController = new AbortController(); - private syncRulesId?: number; private syncRulesContent?: storage.PersistedSyncRulesContent; public storage?: SyncRulesBucketStorage; private settledReplicationPromise?: Promise>; @@ -45,17 +42,15 @@ export class WalStreamTestContext implements AsyncDisposable { } const storageVersion = options?.storageVersion ?? LEGACY_STORAGE_VERSION; - const versionedBuckets = STORAGE_VERSION_CONFIG[storageVersion]?.versionedBuckets ?? false; - return new WalStreamTestContext(f, connectionManager, options?.walStreamOptions, storageVersion, versionedBuckets); + return new WalStreamTestContext(f, connectionManager, options?.walStreamOptions, storageVersion); } constructor( public factory: BucketStorageFactory, public connectionManager: PgManager, private walStreamOptions?: Partial, - private storageVersion: number = LEGACY_STORAGE_VERSION, - private versionedBuckets: boolean = STORAGE_VERSION_CONFIG[storageVersion]?.versionedBuckets ?? false + private storageVersion: number = LEGACY_STORAGE_VERSION ) { createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); @@ -97,7 +92,6 @@ export class WalStreamTestContext implements AsyncDisposable { const syncRules = await this.factory.updateSyncRules( updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion }) ); - this.syncRulesId = syncRules.id; this.syncRulesContent = syncRules; this.storage = this.factory.getInstance(syncRules); return this.storage!; @@ -109,7 +103,6 @@ export class WalStreamTestContext implements AsyncDisposable { throw new Error(`Next sync rules not available`); } - this.syncRulesId = syncRules.id; this.syncRulesContent = syncRules; this.storage = this.factory.getInstance(syncRules); return this.storage!; @@ -121,7 +114,6 @@ export class WalStreamTestContext implements AsyncDisposable { throw new Error(`Active sync rules not available`); } - this.syncRulesId = syncRules.id; this.syncRulesContent = syncRules; this.storage = this.factory.getInstance(syncRules); return this.storage!; @@ -194,35 +186,18 @@ export class WalStreamTestContext implements AsyncDisposable { } async getBucketsDataBatch(buckets: Record, options?: { timeout?: number }) { - let checkpoint = await this.getCheckpoint(options); - const syncRules = this.getSyncRulesContent(); - const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(syncRules, bucket, start)); - return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map)); + const helpers = new StorageDataHelpers(this.storage!, this.getSyncRulesContent()); + const checkpoint = await this.getCheckpoint(options); + return helpers.getBucketsDataBatch(buckets, checkpoint); } /** * This waits for a client checkpoint. */ async getBucketData(bucket: string, start?: InternalOpId | string | undefined, options?: { timeout?: number }) { - start ??= 0n; - if (typeof start == 'string') { - start = BigInt(start); - } - const syncRules = this.getSyncRulesContent(); + const helpers = new StorageDataHelpers(this.storage!, this.getSyncRulesContent()); const checkpoint = await this.getCheckpoint(options); - let map = [bucketRequest(syncRules, bucket, start)]; - let data: OplogEntry[] = []; - while (true) { - const batch = this.storage!.getBucketDataBatch(checkpoint, map); - - const batches = await test_utils.fromAsync(batch); - data = data.concat(batches[0]?.chunkData.data ?? []); - if (batches.length == 0 || !batches[0]!.chunkData.has_more) { - break; - } - map = [bucketRequest(syncRules, bucket, BigInt(batches[0]!.chunkData.next_after))]; - } - return data; + return helpers.getBucketData(bucket, checkpoint, start); } async getChecksums(buckets: string[], options?: { timeout?: number }) { diff --git a/packages/service-core-tests/src/test-utils/StorageDataHelpers.ts b/packages/service-core-tests/src/test-utils/StorageDataHelpers.ts new file mode 100644 index 000000000..20363e5ea --- /dev/null +++ b/packages/service-core-tests/src/test-utils/StorageDataHelpers.ts @@ -0,0 +1,44 @@ +import { + InternalOpId, + OplogEntry, + PersistedSyncRules, + PersistedSyncRulesContent, + SyncRulesBucketStorage +} from '@powersync/service-core'; +import { bucketRequest } from './general-utils.js'; +import { fromAsync } from './stream_utils.js'; + +export class StorageDataHelpers { + storage: SyncRulesBucketStorage; + syncRules: PersistedSyncRulesContent | PersistedSyncRules; + + constructor(storage: SyncRulesBucketStorage, syncRules: PersistedSyncRulesContent | PersistedSyncRules) { + this.storage = storage; + this.syncRules = syncRules; + } + + async getBucketData(bucket: string, checkpoint: InternalOpId, start?: InternalOpId | string | undefined) { + start ??= 0n; + if (typeof start == 'string') { + start = BigInt(start); + } + let map = [bucketRequest(this.syncRules, bucket, start)]; + let data: OplogEntry[] = []; + while (true) { + const batch = this.storage!.getBucketDataBatch(checkpoint, map); + + const batches = await fromAsync(batch); + data = data.concat(batches[0]?.chunkData.data ?? []); + if (batches.length == 0 || !batches[0]!.chunkData.has_more) { + break; + } + map = [bucketRequest(this.syncRules, bucket, BigInt(batches[0]!.chunkData.next_after))]; + } + return data; + } + + async getBucketsDataBatch(buckets: Record, checkpoint: InternalOpId) { + const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(this.syncRules, bucket, start)); + return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map)); + } +} diff --git a/packages/service-core-tests/src/test-utils/test-utils-index.ts b/packages/service-core-tests/src/test-utils/test-utils-index.ts index 1b174d84c..a79c44098 100644 --- a/packages/service-core-tests/src/test-utils/test-utils-index.ts +++ b/packages/service-core-tests/src/test-utils/test-utils-index.ts @@ -1,4 +1,5 @@ export * from './bucket-validation.js'; export * from './general-utils.js'; export * from './MetricsHelper.js'; +export * from './StorageDataHelpers.js'; export * from './stream_utils.js'; diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index d9ddc2475..88a7b0583 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -3,7 +3,6 @@ import { RequestParameters, ScopedParameterLookup, SqliteJsonRow } from '@powers import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; import { bucketRequest } from '../test-utils/test-utils-index.js'; -import { parameterLookupScope } from './util.js'; /** * @example @@ -18,7 +17,6 @@ import { parameterLookupScope } from './util.js'; export function registerDataStorageParameterTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION; - const MYBUCKET_1 = parameterLookupScope('mybucket', '1'); test('save and load parameters', async () => { await using factory = await generateStorageFactory(); @@ -37,6 +35,7 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -69,12 +68,20 @@ bucket_definitions: await writer.commit('1/1'); const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); - expect(parameters).toEqual([ - { - group_id: 'group1a' + const parameters = new RequestParameters(new JwtPayload({ sub: 'user1' }), {}); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; + + const buckets = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => l.indexKey)).toEqual([['user1']]); + + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets).toEqual([{ group_id: 'group1a' }]); + return parameter_sets; } - ]); + }); + + expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'mybucket["group1a"]').bucket]); }); test('it should use the latest version', async () => { @@ -94,6 +101,7 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -121,20 +129,30 @@ bucket_definitions: await writer.commit('1/2'); const checkpoint2 = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint2.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); - expect(parameters).toEqual([ - { - group_id: 'group2' + const parameters = new RequestParameters(new JwtPayload({ sub: 'user1' }), {}); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; + + const buckets1 = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => l.indexKey)).toEqual([['user1']]); + + const parameter_sets = await checkpoint1.getParameterSets(lookups); + expect(parameter_sets).toEqual([{ group_id: 'group1' }]); + return parameter_sets; } - ]); + }); + expect(buckets1.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'mybucket["group1"]').bucket]); - // Use the checkpoint to get older data if relevant - const parameters2 = await checkpoint1.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); - expect(parameters2).toEqual([ - { - group_id: 'group1' + const buckets2 = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => l.indexKey)).toEqual([['user1']]); + + const parameter_sets = await checkpoint2.getParameterSets(lookups); + expect(parameter_sets).toEqual([{ group_id: 'group2' }]); + return parameter_sets; } - ]); + }); + expect(buckets2.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'mybucket["group2"]').bucket]); }); test('it should use the latest version after updates', async () => { @@ -154,6 +172,7 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const table = await test_utils.resolveTestTable(writer, 'todos', ['id', 'list_id'], config); @@ -197,18 +216,28 @@ bucket_definitions: // There removal operation for the association of `list2`::`todo2` should not interfere with the new // association of `list1`::`todo2` const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ - ScopedParameterLookup.direct(MYBUCKET_1, ['list1']), - ScopedParameterLookup.direct(MYBUCKET_1, ['list2']) - ]); + const parameters = new RequestParameters( + new JwtPayload({ sub: 'u1', parameters: { list_id: ['list1', 'list2'] } }), + {} + ); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; - expect(parameters.sort((a, b) => (a.todo_id as string).localeCompare(b.todo_id as string))).toEqual([ - { - todo_id: 'todo1' - }, - { - todo_id: 'todo2' + const buckets = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => JSON.stringify(l.indexKey)).sort()).toEqual(['["list1"]', '["list2"]']); + + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets.sort((a, b) => (a.todo_id as string).localeCompare(b.todo_id as string))).toEqual([ + { todo_id: 'todo1' }, + { todo_id: 'todo2' } + ]); + return parameter_sets; } + }); + + expect(buckets.map((b) => b.bucket).sort()).toEqual([ + bucketRequest(syncRules, 'mybucket["todo1"]').bucket, + bucketRequest(syncRules, 'mybucket["todo2"]').bucket ]); }); @@ -229,6 +258,7 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -248,20 +278,27 @@ bucket_definitions: await writer.commit('1/1'); - const TEST_PARAMS = { group_id: 'group1' }; - const checkpoint = await bucketStorage.getCheckpoint(); + const testQuery = async (jwtParameters: Record, expectedParameterSets: SqliteJsonRow[]) => { + const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: jwtParameters }), {}); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; - const parameters1 = await checkpoint.getParameterSets([ - ScopedParameterLookup.direct(MYBUCKET_1, [314n, 314, 3.14]) + return await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets).toEqual(expectedParameterSets); + return parameter_sets; + } + }); + }; + + expect(await testQuery({ n1: 314n, f2: 314, f3: 3.14 }, [{ group_id: 'group1' }])).toMatchObject([ + { bucket: bucketRequest(syncRules, 'mybucket["group1"]').bucket } ]); - expect(parameters1).toEqual([TEST_PARAMS]); - const parameters2 = await checkpoint.getParameterSets([ - ScopedParameterLookup.direct(MYBUCKET_1, [314, 314n, 3.14]) + expect(await testQuery({ n1: 314, f2: 314n, f3: 3.14 }, [{ group_id: 'group1' }])).toMatchObject([ + { bucket: bucketRequest(syncRules, 'mybucket["group1"]').bucket } ]); - expect(parameters2).toEqual([TEST_PARAMS]); - const parameters3 = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, [314n, 314, 3])]); - expect(parameters3).toEqual([]); + expect(await testQuery({ n1: 314n, f2: 314, f3: 3 }, [])).toEqual([]); }); test('save and load parameters with large numbers', async () => { @@ -285,6 +322,7 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -315,14 +353,23 @@ bucket_definitions: await writer.commit('1/1'); - const TEST_PARAMS = { group_id: 'group1' }; - const checkpoint = await bucketStorage.getCheckpoint(); - const parameters1 = await checkpoint.getParameterSets([ - ScopedParameterLookup.direct(MYBUCKET_1, [1152921504606846976n]) - ]); - expect(parameters1).toEqual([TEST_PARAMS]); + const n1 = 1152921504606846976n; + const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { n1 } }), {}); + + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; + const buckets = await querier.queryDynamicBucketDescriptions({ + getParameterSets: async (lookups) => { + expect(lookups.map((l) => l.indexKey)).toEqual([[n1]]); + + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets).toEqual([{ group_id: 'group1' }]); + return parameter_sets; + } + }); + + expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'mybucket["group1"]').bucket]); }); test('save and load parameters with workspaceId', async () => { @@ -366,7 +413,7 @@ bucket_definitions: const buckets = await querier.queryDynamicBucketDescriptions({ async getParameterSets(lookups) { - expect(lookups).toEqual([ScopedParameterLookup.direct(parameterLookupScope('by_workspace', '1'), ['u1'])]); + expect(lookups.map((l) => l.indexKey)).toEqual([['u1']]); const parameter_sets = await checkpoint.getParameterSets(lookups); expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]); @@ -446,7 +493,7 @@ bucket_definitions: const buckets = await querier.queryDynamicBucketDescriptions({ async getParameterSets(lookups) { - expect(lookups).toEqual([ScopedParameterLookup.direct(parameterLookupScope('by_public_workspace', '1'), [])]); + expect(lookups.map((l) => l.indexKey)).toEqual([[]]); const parameter_sets = await checkpoint.getParameterSets(lookups); parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); @@ -560,10 +607,8 @@ bucket_definitions: } }) ).map((e) => e.bucket); - expect(foundLookups).toEqual([ - ScopedParameterLookup.direct(parameterLookupScope('by_workspace', '1'), []), - ScopedParameterLookup.direct(parameterLookupScope('by_workspace', '2'), ['u1']) - ]); + // Not testing the scope anymore - the exact format depends on storage version + expect(foundLookups.map((l) => l.indexKey)).toEqual([[], ['u1']]); parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]); @@ -591,6 +636,7 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -611,9 +657,19 @@ bucket_definitions: await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); + const parameters = new RequestParameters(new JwtPayload({ sub: 'user1' }), {}); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; + + const buckets = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => l.indexKey)).toEqual([['user1']]); - const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); - expect(parameters).toEqual([]); + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets).toEqual([]); + return parameter_sets; + } + }); + expect(buckets).toEqual([]); }); test('invalidate cached parsed sync rules', async () => { @@ -671,6 +727,7 @@ streams: `) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -688,12 +745,37 @@ streams: await writer.commit('1/1'); const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ - ScopedParameterLookup.direct(parameterLookupScope('lookup', '0'), ['baz']) - ]); - expect(parameters).toEqual([ + const parameters = new RequestParameters(new JwtPayload({ sub: 'baz' }), {}); + const querier = sync_rules.getBucketParameterQuerier({ + ...test_utils.querierOptions(parameters), + streams: { + stream: [ + { + priorityOverride: null, + parameters: null, + opaque_id: 123 + } + ] + } + }).querier; + + const buckets = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => l.indexKey)).toEqual([['baz']]); + + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets).toEqual([{ '0': 'bar' }]); + return parameter_sets; + } + }); + console.log('whatabuckets', buckets); + expect(buckets).toHaveLength(1); + expect(buckets).toMatchObject([ { - '0': 'bar' + bucket: expect.stringMatching(/stream.*\["bar"\]$/), + definition: 'stream', + inclusion_reasons: [{ subscription: 123 }], + priority: 3 } ]); }); diff --git a/packages/service-core/src/storage/BucketStorageFactory.ts b/packages/service-core/src/storage/BucketStorageFactory.ts index 6b68aee45..79a2eb9c0 100644 --- a/packages/service-core/src/storage/BucketStorageFactory.ts +++ b/packages/service-core/src/storage/BucketStorageFactory.ts @@ -161,6 +161,12 @@ export interface UpdateSyncRulesOptions { * compiler. */ plan: SerializedSyncPlan | null; + + /** + * Parsed sync rules version, primarily to generate a definition mapping. + * Not persisted, and the defaultSchema used for parsing is not relevant. + */ + parsed: SyncConfigWithErrors; }; lock?: boolean; storageVersion?: number; @@ -198,10 +204,11 @@ export function updateSyncRulesFromYaml( } export function updateSyncRulesFromConfig( - { config, errors }: SyncConfigWithErrors, + parsed: SyncConfigWithErrors, options?: Omit ): UpdateSyncRulesOptions { let plan: SerializedSyncPlan | null = null; + const { config, errors } = parsed; if (config instanceof PrecompiledSyncConfig) { const eventDescriptors: Record = {}; for (const event of config.eventDescriptors) { @@ -216,7 +223,7 @@ export function updateSyncRulesFromConfig( }; } - return { config: { yaml: config.content, plan }, ...options }; + return { config: { yaml: config.content, plan, parsed }, ...options }; } export interface GetIntanceOptions { diff --git a/packages/service-core/src/storage/PersistedSyncRulesContent.ts b/packages/service-core/src/storage/PersistedSyncRulesContent.ts index 97716b1ba..52de3b457 100644 --- a/packages/service-core/src/storage/PersistedSyncRulesContent.ts +++ b/packages/service-core/src/storage/PersistedSyncRulesContent.ts @@ -144,8 +144,10 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon } asUpdateOptions(options?: Omit): UpdateSyncRulesOptions { + // defaultSchema is not relevant for the parsed version here + const parsed = this.parsed({ defaultSchema: 'not_applicable' }); return { - config: { yaml: this.sync_rules_content, plan: this.compiled_plan }, + config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed: parsed.sync_rules }, ...options }; } diff --git a/packages/service-core/src/storage/bson.ts b/packages/service-core/src/storage/bson.ts index ad7ee3e16..831e1da30 100644 --- a/packages/service-core/src/storage/bson.ts +++ b/packages/service-core/src/storage/bson.ts @@ -40,11 +40,6 @@ export const deserializeParameterLookup = (lookup: bson.Binary) => { return parsed; }; -export const getLookupBucketDefinitionName = (lookup: bson.Binary) => { - const parsed = deserializeParameterLookup(lookup); - return parsed[0] as string; -}; - /** * True if this is a bson.UUID. * diff --git a/packages/sync-rules/src/BucketParameterQuerier.ts b/packages/sync-rules/src/BucketParameterQuerier.ts index de53c482c..b3a676999 100644 --- a/packages/sync-rules/src/BucketParameterQuerier.ts +++ b/packages/sync-rules/src/BucketParameterQuerier.ts @@ -120,6 +120,31 @@ export class ScopedParameterLookup { return (this.#cachedSerializedForm ??= JSONBig.stringify(this.values)); } + /** + * Index id. + * + * This depends on the lookup being constructed with lookupName = indexId, and queryId = ''. + */ + get indexId(): string { + const indexId = this.values[0]; + // TODO: Consider restructuring so that these values aren't present at all + if (this.values[1] != '') { + throw new Error('Unexpected queryId'); + } else if (typeof indexId != 'string') { + throw new Error('Unexpected indexId'); + } + return indexId; + } + + /** + * Returns the "key" portion of the lookup values. + * + * This is this.values, excluding the first "lookupName" and "queryId". + */ + get indexKey(): SqliteJsonValue[] { + return this.values.slice(2); + } + static normalized(scope: ParameterLookupScope, lookup: UnscopedParameterLookup): ScopedParameterLookup { return new ScopedParameterLookup(scope.source, [scope.lookupName, scope.queryId, ...lookup.lookupValues]); } diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index 996de8056..6e07f0a38 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -8,7 +8,11 @@ export interface BucketDataScope { } export interface ParameterLookupScope { - /** The lookup name + queryid is used to reference the parameter lookup record. */ + /** + * The lookup name + queryid is used to reference the parameter lookup record. + * + * In newer storage versions, lookupName = indexId, and queryId = ''. + */ lookupName: string; queryId: string; /** Source used to generate parameter lookups. */