Fix parameter query results logging

.changeset/unlucky-dingos-care.md

@@ -0,0 +1,9 @@
+---
+'@powersync/service-module-postgres-storage': patch
+'@powersync/service-module-mongodb-storage': patch
+'@powersync/service-core': patch
+'@powersync/lib-service-postgres': patch
+'@powersync/lib-services-framework': patch
+---
+
+Fix PSYNC_S2305 logging when parameter limit is exceeded.

libs/lib-postgres/src/db/connection/DatabaseClient.ts

@@ -1,4 +1,5 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
+import { DO_NOT_LOG } from '@powersync/lib-services-framework';
 import * as pgwire from '@powersync/service-jpgwire';
 import { AbstractPostgresConnection, sql } from './AbstractPostgresConnection.js';
 import { ConnectionLease, ConnectionSlot, NotificationListener } from './ConnectionSlot.js';
@@ -31,6 +32,8 @@ export const TRANSACTION_CONNECTION_COUNT = 5;
  * which require being executed on the same connection.
  */
 export class DatabaseClient extends AbstractPostgresConnection<DatabaseClientListener> {
+  [DO_NOT_LOG] = true;
+
   closed: boolean;
 
   pool: pgwire.PgClient;

libs/lib-services/package.json

@@ -27,6 +27,7 @@
     "dotenv": "^16.4.5",
     "ipaddr.js": "^2.1.0",
     "lodash": "^4.17.21",
+    "safe-stable-stringify": "^2.5.0",
     "ts-codec": "^1.3.0",
     "uuid": "^11.1.0",
     "winston": "^3.13.0",

libs/lib-services/src/logger/Logger.ts

@@ -1,4 +1,7 @@
-import winston from 'winston';
+import { ServiceAssertionError } from '@powersync/service-errors';
+
+import jsonStringify from 'safe-stable-stringify';
+import winston, { format } from 'winston';
 
 const prefixFormat = winston.format((info) => {
   if (info.prefix) {
@@ -13,13 +16,60 @@ const prefixFormat = winston.format((info) => {
 export const DEFAULT_LOG_LEVEL = 'info';
 export const DEFAULT_LOG_FORMAT = process.env.NODE_ENV == 'production' ? 'json' : 'text';
 
+/**
+ * Set this field on an object to ensure it is never logged.
+ * This will throw an assertion error if it is logged.
+ */
+export const DO_NOT_LOG = Symbol('DO_NOT_LOG');
+
+/**
+ * Filter potentially noise or sensitive values from logs.
+ *
+ * This throws a hard error if the DO_NOT_LOG Symbol is encountered.
+ */
+const logFilter = (key: string, value: unknown) => {
+  if (value != null && typeof value == 'object' && (value as any)[DO_NOT_LOG]) {
+    throw new ServiceAssertionError(`${Object.getPrototypeOf(value)?.constructor?.name} must not be logged`);
+  }
+  return value;
+};
+
+const MESSAGE = Symbol.for('message');
+
+/**
+ * Like winston.format.simple, but with a custom replacer to filter logs.
+ */
+const filteredSimple = format((info) => {
+  const stringifiedRest = jsonStringify(
+    Object.assign({}, info, {
+      level: undefined,
+      message: undefined,
+      splat: undefined
+    }),
+    logFilter
+  );
+
+  const padding = (info.padding && info.padding[info.level]) || '';
+  if (stringifiedRest !== '{}') {
+    info[MESSAGE] = `${info.level}:${padding} ${info.message} ${stringifiedRest}`;
+  } else {
+    info[MESSAGE] = `${info.level}:${padding} ${info.message}`;
+  }
+
+  return info;
+});
+
 export namespace LogFormat {
   export const development = winston.format.combine(
     prefixFormat(),
     winston.format.colorize({ level: true }),
-    winston.format.simple()
+    filteredSimple()
+  );
+  export const production = winston.format.combine(
+    prefixFormat(),
+    winston.format.timestamp(),
+    winston.format.json({ replacer: logFilter })
   );
-  export const production = winston.format.combine(prefixFormat(), winston.format.timestamp(), winston.format.json());
 }
 
 export const logger = winston.createLogger();

modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts

@@ -1,6 +1,6 @@
 import { GetIntanceOptions, storage } from '@powersync/service-core';
 
-import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import { DO_NOT_LOG, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import { v4 as uuid } from 'uuid';
 
 import * as lib_mongo from '@powersync/lib-service-mongodb';
@@ -18,6 +18,8 @@ export interface MongoBucketStorageOptions {
 }
 
 export class MongoBucketStorage extends storage.BucketStorageFactory {
+  [DO_NOT_LOG] = true;
+
   private readonly client: mongo.MongoClient;
   private readonly session: mongo.ClientSession;
   // TODO: This is still Postgres specific and needs to be reworked

modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -2,6 +2,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 import {
   BaseObserver,
+  DO_NOT_LOG,
   logger,
   ReplicationAbortedError,
   ServiceAssertionError
@@ -67,6 +68,8 @@ export class MongoSyncBucketStorage
   extends BaseObserver<storage.SyncRulesBucketStorageListener>
   implements storage.SyncRulesBucketStorage
 {
+  [DO_NOT_LOG] = true;
+
   private readonly db: VersionedPowerSyncMongo;
   readonly checksums: MongoChecksums;
 
@@ -1043,15 +1046,19 @@ interface InternalCheckpointChanges extends CheckpointChanges {
 }
 
 class MongoReplicationCheckpoint implements ReplicationCheckpoint {
+  #storage: MongoSyncBucketStorage;
+
   constructor(
-    private storage: MongoSyncBucketStorage,
+    storage: MongoSyncBucketStorage,
     public readonly checkpoint: InternalOpId,
     public readonly lsn: string | null,
     public snapshotTime: mongo.Timestamp
-  ) {}
+  ) {
+    this.#storage = storage;
+  }
 
   async getParameterSets(lookups: ScopedParameterLookup[]): Promise<SqliteJsonRow[]> {
-    return this.storage.getParameterSets(this, lookups);
+    return this.#storage.getParameterSets(this, lookups);
   }
 }
 

modules/module-mongodb-storage/src/storage/implementation/db.ts

@@ -2,7 +2,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
 
-import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { DO_NOT_LOG, ServiceAssertionError } from '@powersync/lib-services-framework';
 import { MongoStorageConfig } from '../../types/types.js';
 import {
   BucketDataDocument,
@@ -29,6 +29,8 @@ export interface PowerSyncMongoOptions {
 }
 
 export class PowerSyncMongo {
+  [DO_NOT_LOG] = true;
+
   readonly current_data: mongo.Collection<CurrentDataDocument>;
   readonly v3_current_data: mongo.Collection<CurrentDataDocumentV3>;
   readonly bucket_data: mongo.Collection<BucketDataDocument>;
@@ -206,6 +208,7 @@ export class PowerSyncMongo {
 export class VersionedPowerSyncMongo {
   readonly client: mongo.MongoClient;
   readonly db: mongo.Db;
+  [DO_NOT_LOG] = true;
 
   readonly storageConfig: StorageConfig;
   #upstream: PowerSyncMongo;

modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts

@@ -18,6 +18,7 @@ export type PostgresBucketStorageOptions = {
 };
 
 export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
+  [framework.DO_NOT_LOG] = true;
   readonly db: lib_postgres.DatabaseClient;
   public readonly slot_name_prefix: string;
 

modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts

@@ -50,6 +50,8 @@ export class PostgresSyncRulesStorage
   extends framework.BaseObserver<storage.SyncRulesBucketStorageListener>
   implements storage.SyncRulesBucketStorage
 {
+  [framework.DO_NOT_LOG] = true;
+
   public readonly group_id: number;
   public readonly sync_rules: storage.PersistedSyncRulesContent;
   public readonly slot_name: string;

packages/service-core/src/sync/BucketChecksumState.ts

@@ -517,7 +517,7 @@ export class BucketParameterState {
 
       let errorMessage = error.message;
       const logData: any = {
-        checkpoint: checkpoint,
+        checkpoint: checkpoint.base.checkpoint,
         user_id: this.syncParams.userId,
         parameter_query_results: update.buckets.length
       };

packages/service-core/test/src/sync/BucketChecksumState.test.ts

@@ -985,6 +985,7 @@ bucket_definitions:
       expect(errorMessages[0]).toContain('tasks: 20');
       expect(errorMessages[0]).toContain('comments: 10');
 
+      expect(errorData[0].checkpoint).toEqual(1n);
       expect(errorData[0].parameter_query_results).toBe(60);
       expect(errorData[0].parameter_query_results_by_definition).toEqual({
         projects: 30,