From c00b9e2edb6a8d032ab8b22fe2c8c181e6ecaab1 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 5 Mar 2024 17:05:57 +0200 Subject: [PATCH 1/5] Reduce lodash usage. --- .../src/client/AbstractPowerSyncDatabase.ts | 48 ++++++++++--------- .../src/client/sync/bucket/CrudEntry.ts | 4 +- .../AbstractStreamingSyncImplementation.ts | 24 ++++++---- .../src/db/crud/SyncStatus.ts | 4 +- .../src/db/schema/Table.ts | 12 +++-- ...AbstractWebPowerSyncDatabaseOpenFactory.ts | 13 +++-- .../sync/SSRWebStreamingSyncImplementation.ts | 1 - .../SharedWebStreamingSyncImplementation.ts | 1 - .../db/sync/WebStreamingSyncImplementation.ts | 1 - .../src/worker/db/open-db.ts | 26 ++++------ .../worker/sync/SharedSyncImplementation.ts | 6 ++- 11 files changed, 73 insertions(+), 67 deletions(-) diff --git a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts index b1ac652cd..8174ad0db 100644 --- a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts +++ b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts @@ -1,24 +1,25 @@ -import _ from 'lodash'; import { Mutex } from 'async-mutex'; +import { EventIterator } from 'event-iterator'; import Logger, { ILogger } from 'js-logger'; +import intersection from 'lodash/intersection'; +import throttle from 'lodash/throttle'; import { DBAdapter, QueryResult, Transaction, isBatchedUpdateNotification } from '../db/DBAdapter'; -import { Schema } from '../db/schema/Schema'; import { SyncStatus } from '../db/crud/SyncStatus'; import { UploadQueueStats } from '../db/crud/UploadQueueStatus'; +import { Schema } from '../db/schema/Schema'; +import { BaseObserver } from '../utils/BaseObserver'; +import { mutexRunExclusive } from '../utils/mutex'; +import { quoteIdentifier } from '../utils/strings'; import { PowerSyncBackendConnector } from './connection/PowerSyncBackendConnector'; +import { BucketStorageAdapter, PSInternalTable } from './sync/bucket/BucketStorageAdapter'; +import { CrudBatch } from './sync/bucket/CrudBatch'; +import { CrudEntry, CrudEntryJSON } from './sync/bucket/CrudEntry'; +import { CrudTransaction } from './sync/bucket/CrudTransaction'; import { AbstractStreamingSyncImplementation, DEFAULT_CRUD_UPLOAD_THROTTLE_MS, StreamingSyncImplementationListener } from './sync/stream/AbstractStreamingSyncImplementation'; -import { CrudBatch } from './sync/bucket/CrudBatch'; -import { CrudTransaction } from './sync/bucket/CrudTransaction'; -import { BucketStorageAdapter, PSInternalTable } from './sync/bucket/BucketStorageAdapter'; -import { CrudEntry, CrudEntryJSON } from './sync/bucket/CrudEntry'; -import { mutexRunExclusive } from '../utils/mutex'; -import { BaseObserver } from '../utils/BaseObserver'; -import { EventIterator } from 'event-iterator'; -import { quoteIdentifier } from '../utils/strings'; export interface DisconnectAndClearOptions { /** When set to false, data in local-only tables is preserved. */ @@ -538,25 +539,26 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver { - //Fetch initial data + // Fetch initial data yield await this.executeReadOnly(sql, parameters); - const resolvedTables = options?.tables ?? []; + const resolvedTables = options?.tables ? [...options.tables] : []; if (!options?.tables) { const explained = await this.getAll<{ opcode: string; p3: number; p2: number }>(`EXPLAIN ${sql}`, parameters); - const rootPages = _.chain(explained) - .filter((row) => row['opcode'] == 'OpenRead' && row['p3'] == 0 && _.isNumber(row['p2'])) - .map((row) => row['p2']) - .value(); + const rootPages = explained + .filter((row) => row.opcode == 'OpenRead' && row.p3 == 0 && typeof row.p2 == 'number') + .map((row) => row.p2); const tables = await this.getAll<{ tbl_name: string }>( - `SELECT tbl_name FROM sqlite_master WHERE rootpage IN (SELECT json_each.value FROM json_each(?))`, + `SELECT DISTINCT tbl_name FROM sqlite_master WHERE rootpage IN (SELECT json_each.value FROM json_each(?))`, [JSON.stringify(rootPages)] ); - tables.forEach((t) => resolvedTables.push(t.tbl_name.replace(POWERSYNC_TABLE_MATCH, ''))); + for (let table of tables) { + resolvedTables.push(table.tbl_name.replace(POWERSYNC_TABLE_MATCH, '')); + } } for await (const event of this.onChange({ ...(options ?? {}), - tables: _.uniq(resolvedTables) + tables: resolvedTables })) { yield await this.executeReadOnly(sql, parameters); } @@ -578,12 +580,12 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver((eventOptions) => { - const flushTableUpdates = _.throttle( + const flushTableUpdates = throttle( () => { - const intersection = _.intersection(watchedTables, throttledTableUpdates); - if (intersection.length) { + const changedTables = intersection(watchedTables, throttledTableUpdates); + if (changedTables.length > 0) { eventOptions.push({ - changedTables: intersection + changedTables }); } throttledTableUpdates = []; diff --git a/packages/powersync-sdk-common/src/client/sync/bucket/CrudEntry.ts b/packages/powersync-sdk-common/src/client/sync/bucket/CrudEntry.ts index 64ef48ea1..53d51277c 100644 --- a/packages/powersync-sdk-common/src/client/sync/bucket/CrudEntry.ts +++ b/packages/powersync-sdk-common/src/client/sync/bucket/CrudEntry.ts @@ -1,4 +1,4 @@ -import _ from 'lodash'; +import isEqual from 'lodash/isEqual'; /** * 64-bit unsigned integer stored as a string in base-10. @@ -109,7 +109,7 @@ export class CrudEntry { } equals(entry: CrudEntry) { - return _.isEqual(this.toComparisonArray(), entry.toComparisonArray()); + return isEqual(this.toComparisonArray(), entry.toComparisonArray()); } /** diff --git a/packages/powersync-sdk-common/src/client/sync/stream/AbstractStreamingSyncImplementation.ts b/packages/powersync-sdk-common/src/client/sync/stream/AbstractStreamingSyncImplementation.ts index 61afabb70..58b9c0360 100644 --- a/packages/powersync-sdk-common/src/client/sync/stream/AbstractStreamingSyncImplementation.ts +++ b/packages/powersync-sdk-common/src/client/sync/stream/AbstractStreamingSyncImplementation.ts @@ -1,4 +1,6 @@ -import _ from 'lodash'; +import throttle from 'lodash/throttle'; +import defer from 'lodash/defer'; + import Logger, { ILogger } from 'js-logger'; import { @@ -76,7 +78,7 @@ export abstract class AbstractStreamingSyncImplementation extends BaseObserver { if (!this.syncStatus.connected || this.syncStatus.dataFlowStatus.uploading) { return; @@ -206,6 +208,7 @@ export abstract class AbstractStreamingSyncImplementation extends BaseObserver this.triggerCrudUpload()); + defer(() => this.triggerCrudUpload()); this.updateSyncStatus({ connected: true }); @@ -256,7 +259,7 @@ export abstract class AbstractStreamingSyncImplementation extends BaseObserver !InvalidSQLCharacters.test(name)) - .value(); + if (InvalidSQLCharacters.test(this.name)) { + return false; + } + if (this.viewNameOverride != null && InvalidSQLCharacters.test(this.viewNameOverride)) { + return false; + } + return true; } validate() { diff --git a/packages/powersync-sdk-web/src/db/adapters/AbstractWebPowerSyncDatabaseOpenFactory.ts b/packages/powersync-sdk-web/src/db/adapters/AbstractWebPowerSyncDatabaseOpenFactory.ts index e4caf2c70..e3e3f95ce 100644 --- a/packages/powersync-sdk-web/src/db/adapters/AbstractWebPowerSyncDatabaseOpenFactory.ts +++ b/packages/powersync-sdk-web/src/db/adapters/AbstractWebPowerSyncDatabaseOpenFactory.ts @@ -1,4 +1,3 @@ -import _ from 'lodash'; import { AbstractPowerSyncDatabase, AbstractPowerSyncDatabaseOpenFactory, @@ -73,10 +72,14 @@ export abstract class AbstractWebPowerSyncDatabaseOpenFactory extends AbstractPo } protected resolveDBFlags(): WebPowerSyncFlags { - return _.merge(_.clone(DEFAULT_POWERSYNC_FLAGS), { - ssrMode: this.isServerSide(), - enableMultiTabs: this.options.flags?.enableMultiTabs - }); + let flags = { + ...DEFAULT_POWERSYNC_FLAGS, + ssrMode: this.isServerSide() + }; + if (typeof this.options.flags?.enableMultiTabs != 'undefined') { + flags.enableMultiTabs = this.options.flags.enableMultiTabs; + } + return flags; } generateInstance(options: PowerSyncDatabaseOptions): AbstractPowerSyncDatabase { diff --git a/packages/powersync-sdk-web/src/db/sync/SSRWebStreamingSyncImplementation.ts b/packages/powersync-sdk-web/src/db/sync/SSRWebStreamingSyncImplementation.ts index 6b084bc6a..484718f78 100644 --- a/packages/powersync-sdk-web/src/db/sync/SSRWebStreamingSyncImplementation.ts +++ b/packages/powersync-sdk-web/src/db/sync/SSRWebStreamingSyncImplementation.ts @@ -1,4 +1,3 @@ -import _ from 'lodash'; import { AbstractStreamingSyncImplementation, AbstractStreamingSyncImplementationOptions, diff --git a/packages/powersync-sdk-web/src/db/sync/SharedWebStreamingSyncImplementation.ts b/packages/powersync-sdk-web/src/db/sync/SharedWebStreamingSyncImplementation.ts index 826162941..c6bb44782 100644 --- a/packages/powersync-sdk-web/src/db/sync/SharedWebStreamingSyncImplementation.ts +++ b/packages/powersync-sdk-web/src/db/sync/SharedWebStreamingSyncImplementation.ts @@ -1,4 +1,3 @@ -import _ from 'lodash'; import { v4 as uuid } from 'uuid'; import * as Comlink from 'comlink'; import { diff --git a/packages/powersync-sdk-web/src/db/sync/WebStreamingSyncImplementation.ts b/packages/powersync-sdk-web/src/db/sync/WebStreamingSyncImplementation.ts index 076e02cdb..1fb7d1c27 100644 --- a/packages/powersync-sdk-web/src/db/sync/WebStreamingSyncImplementation.ts +++ b/packages/powersync-sdk-web/src/db/sync/WebStreamingSyncImplementation.ts @@ -1,4 +1,3 @@ -import _ from 'lodash'; import { AbstractStreamingSyncImplementation, AbstractStreamingSyncImplementationOptions, diff --git a/packages/powersync-sdk-web/src/worker/db/open-db.ts b/packages/powersync-sdk-web/src/worker/db/open-db.ts index 7d0cf5314..269ad7487 100644 --- a/packages/powersync-sdk-web/src/worker/db/open-db.ts +++ b/packages/powersync-sdk-web/src/worker/db/open-db.ts @@ -1,6 +1,5 @@ import * as SQLite from '@journeyapps/wa-sqlite'; import '@journeyapps/wa-sqlite'; -import _ from 'lodash'; import * as Comlink from 'comlink'; import { v4 as uuid } from 'uuid'; import { QueryResult } from '@journeyapps/powersync-sdk-common'; @@ -93,21 +92,16 @@ export async function _openDB(dbFileName: string): Promise { } } - const rows = _.chain(results) - .filter(({ rows }) => !!rows.length) - .flatMap(({ columns, rows }) => - _.map(rows, (row) => - _.reduce( - columns, - (out: Record, key: string, index) => { - out[key] = row[index]; - return out; - }, - {} - ) - ) - ) - .value(); + let rows: Record[] = []; + for (let resultset of results) { + for (let row of resultset.rows) { + let outRow: Record = {}; + resultset.columns.forEach((key, index) => { + outRow[key] = row[index]; + }); + rows.push(outRow); + } + } const result = { insertId: sqlite3.last_insert_id(db), diff --git a/packages/powersync-sdk-web/src/worker/sync/SharedSyncImplementation.ts b/packages/powersync-sdk-web/src/worker/sync/SharedSyncImplementation.ts index 81d960b4f..cd3a00f7e 100644 --- a/packages/powersync-sdk-web/src/worker/sync/SharedSyncImplementation.ts +++ b/packages/powersync-sdk-web/src/worker/sync/SharedSyncImplementation.ts @@ -1,4 +1,3 @@ -import _ from 'lodash'; import { BaseListener, BaseObserver, SyncStatusOptions } from '@journeyapps/powersync-sdk-common'; export enum SharedSyncMessageType { @@ -38,7 +37,10 @@ export class SharedSyncImplementation extends BaseObserver cb.statusChanged?.(status)); } } From 52afd2bb78e78e1eb9818cfa8c8c926a9292b097 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 5 Mar 2024 17:16:57 +0200 Subject: [PATCH 2/5] Remove more usages of lodash. --- .../src/client/AbstractPowerSyncDatabase.ts | 16 ++++++++-------- .../src/client/sync/bucket/CrudEntry.ts | 4 +--- .../AbstractStreamingSyncImplementation.ts | 3 +-- .../src/db/crud/SyncStatus.ts | 4 +--- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts index 8174ad0db..8327a999a 100644 --- a/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts +++ b/packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts @@ -1,7 +1,6 @@ import { Mutex } from 'async-mutex'; import { EventIterator } from 'event-iterator'; import Logger, { ILogger } from 'js-logger'; -import intersection from 'lodash/intersection'; import throttle from 'lodash/throttle'; import { DBAdapter, QueryResult, Transaction, isBatchedUpdateNotification } from '../db/DBAdapter'; import { SyncStatus } from '../db/crud/SyncStatus'; @@ -574,21 +573,20 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver { const resolvedOptions = options ?? {}; - const watchedTables = resolvedOptions.tables ?? []; + const watchedTables = new Set(resolvedOptions.tables ?? []); - let throttledTableUpdates: string[] = []; + let changedTables = new Set(); const throttleMs = resolvedOptions.throttleMs ?? DEFAULT_WATCH_THROTTLE_MS; return new EventIterator((eventOptions) => { const flushTableUpdates = throttle( () => { - const changedTables = intersection(watchedTables, throttledTableUpdates); - if (changedTables.length > 0) { + if (changedTables.size > 0) { eventOptions.push({ - changedTables + changedTables: [...changedTables] }); } - throttledTableUpdates = []; + changedTables.clear(); }, throttleMs, { leading: false, trailing: true } @@ -610,7 +608,9 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver t.replace(POWERSYNC_TABLE_MATCH, '')); - throttledTableUpdates.push(...mappedTableNames); + for (let table of mappedTableNames) { + changedTables.add(table); + } flushTableUpdates(); } diff --git a/packages/powersync-sdk-common/src/client/sync/bucket/CrudEntry.ts b/packages/powersync-sdk-common/src/client/sync/bucket/CrudEntry.ts index 53d51277c..cb3c1cd1a 100644 --- a/packages/powersync-sdk-common/src/client/sync/bucket/CrudEntry.ts +++ b/packages/powersync-sdk-common/src/client/sync/bucket/CrudEntry.ts @@ -1,5 +1,3 @@ -import isEqual from 'lodash/isEqual'; - /** * 64-bit unsigned integer stored as a string in base-10. * @@ -109,7 +107,7 @@ export class CrudEntry { } equals(entry: CrudEntry) { - return isEqual(this.toComparisonArray(), entry.toComparisonArray()); + return JSON.stringify(this.toComparisonArray()) == JSON.stringify(entry.toComparisonArray()); } /** diff --git a/packages/powersync-sdk-common/src/client/sync/stream/AbstractStreamingSyncImplementation.ts b/packages/powersync-sdk-common/src/client/sync/stream/AbstractStreamingSyncImplementation.ts index 58b9c0360..9513b84a3 100644 --- a/packages/powersync-sdk-common/src/client/sync/stream/AbstractStreamingSyncImplementation.ts +++ b/packages/powersync-sdk-common/src/client/sync/stream/AbstractStreamingSyncImplementation.ts @@ -1,5 +1,4 @@ import throttle from 'lodash/throttle'; -import defer from 'lodash/defer'; import Logger, { ILogger } from 'js-logger'; @@ -226,7 +225,7 @@ export abstract class AbstractStreamingSyncImplementation extends BaseObserver this.triggerCrudUpload()); + Promise.resolve().then(() => this.triggerCrudUpload()); this.updateSyncStatus({ connected: true }); diff --git a/packages/powersync-sdk-common/src/db/crud/SyncStatus.ts b/packages/powersync-sdk-common/src/db/crud/SyncStatus.ts index 6d93f22fc..c12bc1cfe 100644 --- a/packages/powersync-sdk-common/src/db/crud/SyncStatus.ts +++ b/packages/powersync-sdk-common/src/db/crud/SyncStatus.ts @@ -1,5 +1,3 @@ -import isEqual from 'lodash/isEqual'; - export type SyncDataFlowStatus = Partial<{ downloading: boolean; uploading: boolean; @@ -49,7 +47,7 @@ export class SyncStatus { } isEqual(status: SyncStatus) { - return isEqual(this.options, status.options); + return JSON.stringify(this.options) == JSON.stringify(status.options); } getMessage() { From 71b817578f52084eca25d479622b0bac8d735077 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 5 Mar 2024 17:34:18 +0200 Subject: [PATCH 3/5] Fix test. --- packages/powersync-sdk-web/tests/crud.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/powersync-sdk-web/tests/crud.test.ts b/packages/powersync-sdk-web/tests/crud.test.ts index cc2499057..aa1350c81 100644 --- a/packages/powersync-sdk-web/tests/crud.test.ts +++ b/packages/powersync-sdk-web/tests/crud.test.ts @@ -163,8 +163,8 @@ describe('CRUD Tests', () => { const tx = (await powersync.getNextCrudTransaction())!; expect(tx.transactionId).equals(1); const expectedCrudEntry = new CrudEntry(1, UpdateType.PUT, 'logs', testId, 1, { - level: 'INFO', - content: 'test log' + content: 'test log', + level: 'INFO' }); expect(tx.crud[0].equals(expectedCrudEntry)).true; }); From 1bc01a2a0189d4729db51a28e6ccd0e66043b27a Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 7 Mar 2024 16:22:34 +0200 Subject: [PATCH 4/5] Deep merge status updates. --- .../src/worker/sync/SharedSyncImplementation.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/powersync-sdk-web/src/worker/sync/SharedSyncImplementation.ts b/packages/powersync-sdk-web/src/worker/sync/SharedSyncImplementation.ts index cd3a00f7e..40d1c324e 100644 --- a/packages/powersync-sdk-web/src/worker/sync/SharedSyncImplementation.ts +++ b/packages/powersync-sdk-web/src/worker/sync/SharedSyncImplementation.ts @@ -39,7 +39,11 @@ export class SharedSyncImplementation extends BaseObserver cb.statusChanged?.(status)); } From 77b3078756bcc3fd61d505c405f4b5916aed338a Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 12 Mar 2024 10:07:46 +0200 Subject: [PATCH 5/5] Add changeset. --- .changeset/mean-numbers-remember.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/mean-numbers-remember.md diff --git a/.changeset/mean-numbers-remember.md b/.changeset/mean-numbers-remember.md new file mode 100644 index 000000000..a614afa5d --- /dev/null +++ b/.changeset/mean-numbers-remember.md @@ -0,0 +1,6 @@ +--- +"@journeyapps/powersync-sdk-common": patch +"@journeyapps/powersync-sdk-web": patch +--- + +Reduce JS bundle size