diff --git a/CHANGELOG-Unreleased.md b/CHANGELOG-Unreleased.md index 97df9c0cd..a641a6196 100644 --- a/CHANGELOG-Unreleased.md +++ b/CHANGELOG-Unreleased.md @@ -6,11 +6,17 @@ ### New features +- [LokiJS] New `onPersistenceError` option: called when saving the database to IndexedDB fails (e.g. quota exceeded, aborted transaction), once per failure. Quota exceeded save errors are now also passed to `onQuotaExceededError` +- [LokiJS] New `allowInMemoryFallback` option (default: `true`). Pass `false` to make setup fail (and `onSetUpError` be called) instead of silently falling back to an in-memory database when IndexedDB is unavailable +- [LokiJS] New `onInMemoryFallback` option: called when the database falls back to being in-memory only - New `Collection.observeRecord(id)`: a failsafe version of `findAndObserve(id)` that emits `null` while the record does not exist (instead of erroring), emits the record once it is created, and emits `null` (instead of completing) when it is deleted. `@experimentalFailsafe` relations now use it, so their `observe()` recovers when a missing related record is created later ### Fixes - [LokiJS] Multitab sync issue fix +- [LokiJS] When IndexedDB database is deleted in another tab (`onversionchange`), the database is no longer re-created from this tab's memory on next autosave. Instead, the adapter stops saving, rejects further changes, and calls `onPersistenceError` +- [LokiJS] Failures to save the database to IndexedDB are now always logged +- [LokiJS] When accessing or opening IndexedDB throws synchronously (e.g. `SecurityError` when storage access is denied), the adapter now falls back to an in-memory database like for other IndexedDB open failures (or fails setup with `allowInMemoryFallback: false`), instead of always failing setup - [Android] Added linker flag for building with 16kB page alignment - [TS] make catchError visible to typescript - [LokiJS] Fixed `unsafeResetDatabase()` failing with "another operation is already in progress" when it's called while an autosave is in progress diff --git a/docs-website/docs/docs/Setup.md b/docs-website/docs/docs/Setup.md index 88b940c2e..19569cea9 100644 --- a/docs-website/docs/docs/Setup.md +++ b/docs-website/docs/docs/Setup.md @@ -89,6 +89,17 @@ const adapter = new LokiJSAdapter({ onSetUpError: (error) => { // Database failed to load -- offer the user to reload the app or log out }, + onPersistenceError: (error) => { + // Saving to IndexedDB failed (e.g. disk full, database deleted in another tab). Changes made + // since the last successful save only exist in memory and will be lost if the page is closed. + // Called once per failure, not on every autosave retry + }, + // (true by default) Pass `false` to make setup fail (and call `onSetUpError`) instead of silently + // falling back to an in-memory database when IndexedDB is unavailable (e.g. some private modes) + allowInMemoryFallback: true, + onInMemoryFallback: (reason) => { + // IndexedDB is unavailable, so data will be lost when the page is closed -- warn the user + }, extraIncrementalIDBOptions: { onDidOverwrite: () => { // Called when this adapter is forced to overwrite contents of IndexedDB. @@ -99,6 +110,7 @@ const adapter = new LokiJSAdapter({ onversionchange: () => { // database was deleted in another browser tab (user logged out), so we must make sure we delete // it in this tab as well - usually best to just refresh the page + // (from this point, this tab no longer saves to IndexedDB and rejects further changes) if (checkIfUserIsLoggedIn()) { window.location.reload() } @@ -109,6 +121,16 @@ const adapter = new LokiJSAdapter({ // The rest is the same! ``` +:::note +On the web, `database.write()` resolves once changes are applied in memory. LokiJS saves them to IndexedDB shortly after (autosave), so a failure to save is only reported via `onPersistenceError`. + +Multiple tabs of the same app share one IndexedDB database, but each tab keeps its own in-memory copy: + +- When another tab deletes the database (e.g. on logout), this tab stops saving to IndexedDB, rejects further changes (including `unsafeExecute`), and calls `onPersistenceError` and `onversionchange`. Reload the page to continue. +- When two tabs make changes concurrently, the last tab to save overwrites changes saved by the other one (`onDidOverwrite` is called). WatermelonDB does not coordinate writers across tabs. +- `onPersistenceError`, `onInMemoryFallback`, `onQuotaExceededError` and `extraIncrementalIDBOptions` callbacks only work with `useWebWorker: false`. +::: + * * * ## Next steps diff --git a/src/adapters/lokijs/index.d.ts b/src/adapters/lokijs/index.d.ts index e09fb142f..6a0a2b1e4 100644 --- a/src/adapters/lokijs/index.d.ts +++ b/src/adapters/lokijs/index.d.ts @@ -35,6 +35,21 @@ export type LokiAdapterOptions = $Exact<{ // This means that app can't save more data or that it will fall back to using in-memory database only // Note that this only works when `useWebWorker: false` onQuotaExceededError?: (error: Error) => void + // Called when saving the database to IndexedDB fails (e.g. quota exceeded, transaction aborted, + // database closed). Changes made since the last successful save are only kept in memory until + // a later save succeeds, and will be lost if the page is closed. Note that `database.write()` + // resolving does not mean that changes were saved to disk. Called once per failure, not on every + // retry, and called again if saving fails after it succeeded in the meantime. Quota exceeded + // errors are also passed to `onQuotaExceededError`. + // Note that this only works when `useWebWorker: false` + onPersistenceError?: (error: Error) => void + // (true by default) If IndexedDB is not available or fails to open (e.g. in some browsers' private + // mode), the database falls back to being in-memory only and data is lost when the page is closed. + // Pass `false` to make setup fail instead (`onSetUpError` will be called) + allowInMemoryFallback?: boolean + // Called when the database falls back to being in-memory only (see `allowInMemoryFallback`) + // Note that this only works when `useWebWorker: false` + onInMemoryFallback?: (reason: Error) => void // extra options passed to Loki constructor extraLokiOptions?: $Exact<{ autosave?: boolean @@ -47,6 +62,8 @@ export type LokiAdapterOptions = $Exact<{ // You might use it as an opportunity to alert user to the potential loss of data onDidOverwrite?: () => void // Called when internal IndexedDB version changed (most likely the database was deleted in another browser tab) + // When this happens, the database stops being saved (so that it's not re-created from memory), + // further changes are rejected, and `onPersistenceError` is called. // Pass a callback to force log out in this copy of the app as well // (Due to a race condition, it's usually best to just reload the web app) // Note that this only works when not using web workers diff --git a/src/adapters/lokijs/index.js b/src/adapters/lokijs/index.js index 1b32021a6..9d46042b6 100644 --- a/src/adapters/lokijs/index.js +++ b/src/adapters/lokijs/index.js @@ -39,6 +39,21 @@ export type LokiAdapterOptions = $Exact<{ // This means that app can't save more data or that it will fall back to using in-memory database only // Note that this only works when `useWebWorker: false` onQuotaExceededError?: (error: Error) => void, + // Called when saving the database to IndexedDB fails (e.g. quota exceeded, transaction aborted, + // database closed). Changes made since the last successful save are only kept in memory until + // a later save succeeds, and will be lost if the page is closed. Note that `database.write()` + // resolving does not mean that changes were saved to disk. Called once per failure, not on every + // retry, and called again if saving fails after it succeeded in the meantime. Quota exceeded + // errors are also passed to `onQuotaExceededError`. + // Note that this only works when `useWebWorker: false` + onPersistenceError?: (error: Error) => void, + // (true by default) If IndexedDB is not available or fails to open (e.g. in some browsers' private + // mode), the database falls back to being in-memory only and data is lost when the page is closed. + // Pass `false` to make setup fail instead (`onSetUpError` will be called) + allowInMemoryFallback?: boolean, + // Called when the database falls back to being in-memory only (see `allowInMemoryFallback`) + // Note that this only works when `useWebWorker: false` + onInMemoryFallback?: (reason: Error) => void, // extra options passed to Loki constructor extraLokiOptions?: $Exact<{ autosave?: boolean, @@ -51,6 +66,8 @@ export type LokiAdapterOptions = $Exact<{ // You might use it as an opportunity to alert user to the potential loss of data onDidOverwrite?: () => void, // Called when internal IndexedDB version changed (most likely the database was deleted in another browser tab) + // When this happens, the database stops being saved (so that it's not re-created from memory), + // further changes are rejected, and `onPersistenceError` is called. // Pass a callback to force log out in this copy of the app as well // (Due to a race condition, it's usually best to just reload the web app) // Note that this only works when not using web workers diff --git a/src/adapters/lokijs/persistence.test.js b/src/adapters/lokijs/persistence.test.js new file mode 100644 index 000000000..9f8aed9bb --- /dev/null +++ b/src/adapters/lokijs/persistence.test.js @@ -0,0 +1,343 @@ +import { testSchema, mockTaskRaw } from '../__tests__/helpers' + +import LokiJSAdapter from './index' +import DatabaseAdapterCompat from '../compat' + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +const waitFor = async (predicate) => { + for (let i = 0; i < 200; i += 1) { + if (predicate()) { + return + } + // eslint-disable-next-line no-await-in-loop + await sleep(5) + } + throw new Error('Timed out waiting for condition') +} + +// Mimics lokijs's IncrementalIndexedDBAdapter: incremental mode, `options` object, async save callback +class FakeIncrementalAdapter { + mode = 'incremental' + + options = {} + + saveError = null + + saves = 0 + + deletes = 0 + + loadDatabase(dbname, callback) { + callback(null) + } + + saveDatabase(dbname, getLokiCopy, callback) { + getLokiCopy() + this.saves += 1 + const error = this.saveError + setTimeout(() => callback(error), 0) + } + + deleteDatabase(dbname, callback) { + this.deletes += 1 + callback({ success: true }) + } +} + +// What IncrementalIndexedDBAdapter passes to the save callback when an IDB transaction aborts +const abortEvent = (error) => ({ type: 'abort', target: { error } }) + +const quotaError = () => { + const error = new Error('The quota has been exceeded.') + error.name = 'QuotaExceededError' + return error +} + +let adapters = [] + +const makeAdapter = (options) => { + const adapter = new LokiJSAdapter({ + dbName: `test${Math.random()}`, + schema: testSchema, + useWebWorker: false, + useIncrementalIndexedDB: true, + extraLokiOptions: { autosaveInterval: 5 }, + ...options, + }) + adapters.push(adapter) + return adapter +} + +let taskId = 0 +const writeTask = (adapter) => { + taskId += 1 + return adapter.batch([['create', 'tasks', mockTaskRaw({ id: `t${taskId}` })]]) +} + +afterEach(() => { + adapters.forEach((adapter) => { + const loki = adapter._driver?.loki + loki && loki.autosaveDisable() + }) + adapters = [] + delete global.indexedDB +}) + +describe('LokiJSAdapter persistence errors', () => { + it('reports a failing save once, and again after a successful save in between', async () => { + const persistenceAdapter = new FakeIncrementalAdapter() + const onPersistenceError = jest.fn() + const onQuotaExceededError = jest.fn() + const adapter = new DatabaseAdapterCompat( + makeAdapter({ + _testLokiAdapter: persistenceAdapter, + onPersistenceError, + onQuotaExceededError, + }), + ) + + // saves fine + await writeTask(adapter) + await waitFor(() => persistenceAdapter.saves >= 1) + expect(onPersistenceError).not.toHaveBeenCalled() + + // quota exceeded, reported once even though autosave keeps retrying + const error = quotaError() + persistenceAdapter.saveError = abortEvent(error) + await writeTask(adapter) + await waitFor(() => onPersistenceError.mock.calls.length === 1) + const savesAfterFailure = persistenceAdapter.saves + await waitFor(() => persistenceAdapter.saves >= savesAfterFailure + 3) + expect(onPersistenceError).toHaveBeenCalledTimes(1) + // raw IDB event is unwrapped + expect(onPersistenceError).toHaveBeenCalledWith(error) + expect(onQuotaExceededError).toHaveBeenCalledTimes(1) + expect(onQuotaExceededError).toHaveBeenCalledWith(error) + + // recovers - the retried save succeeds + persistenceAdapter.saveError = null + const savesBeforeRecovery = persistenceAdapter.saves + await waitFor(() => persistenceAdapter.saves > savesBeforeRecovery) + await sleep(20) + + // fails again - reported again + const otherError = new Error('Error saving database') + persistenceAdapter.saveError = otherError + await writeTask(adapter) + await waitFor(() => onPersistenceError.mock.calls.length === 2) + expect(onPersistenceError).toHaveBeenLastCalledWith(otherError) + expect(onQuotaExceededError).toHaveBeenCalledTimes(1) + }) + + it('stops saving and breaks the driver when IndexedDB is deleted by another connection', async () => { + const persistenceAdapter = new FakeIncrementalAdapter() + const onPersistenceError = jest.fn() + const onversionchange = jest.fn() + const _onFatalError = jest.fn() + const lokiAdapter = makeAdapter({ + _testLokiAdapter: persistenceAdapter, + onPersistenceError, + _onFatalError, + extraIncrementalIDBOptions: { onversionchange }, + }) + const adapter = new DatabaseAdapterCompat(lokiAdapter) + + await writeTask(adapter) + await waitFor(() => persistenceAdapter.saves >= 1) + + const event = { type: 'versionchange' } + persistenceAdapter.options.onversionchange(event) + + expect(onversionchange).toHaveBeenCalledWith(event) + expect(onPersistenceError).toHaveBeenCalledTimes(1) + expect(onPersistenceError.mock.calls[0][0].name).toBe('VersionChangeError') + expect(_onFatalError).toHaveBeenCalledTimes(1) + + // changes are rejected, and nothing is saved anymore + const savesBefore = persistenceAdapter.saves + await expect(writeTask(adapter)).rejects.toThrow('broken state') + const rawWrite = jest.fn() + await expect(adapter.unsafeExecute({ loki: rawWrite })).rejects.toThrow('broken state') + expect(rawWrite).not.toHaveBeenCalled() + expect(() => lokiAdapter._driver.loki.saveDatabase()).toThrow() + await sleep(30) + expect(persistenceAdapter.saves).toBe(savesBefore) + expect(onPersistenceError).toHaveBeenCalledTimes(1) + + // reads still work + expect(await adapter.find('tasks', `t${taskId - 1}`)).toBeTruthy() + + // database can't be deleted and re-created + const deletesBefore = persistenceAdapter.deletes + await expect(adapter.unsafeResetDatabase()).rejects.toThrow('broken state') + expect(persistenceAdapter.deletes).toBe(deletesBefore) + }) + + it('breaks the driver before notifying, and notifies even if callbacks throw', async () => { + const persistenceAdapter = new FakeIncrementalAdapter() + const onversionchange = jest.fn() + let wasBrokenWhenNotified = false + const lokiAdapter = makeAdapter({ + _testLokiAdapter: persistenceAdapter, + _onFatalError: () => { + throw new Error('fatal error handler failed') + }, + onPersistenceError: () => { + wasBrokenWhenNotified = lokiAdapter._driver._isBroken + throw new Error('handler failed') + }, + extraIncrementalIDBOptions: { onversionchange }, + }) + const adapter = new DatabaseAdapterCompat(lokiAdapter) + await writeTask(adapter) + + persistenceAdapter.options.onversionchange({ type: 'versionchange' }) + expect(wasBrokenWhenNotified).toBe(true) + expect(onversionchange).toHaveBeenCalledTimes(1) + await expect(writeTask(adapter)).rejects.toThrow('broken state') + }) + + it('reports a failing save to onPersistenceError even if onQuotaExceededError throws', async () => { + const persistenceAdapter = new FakeIncrementalAdapter() + const onPersistenceError = jest.fn() + const adapter = new DatabaseAdapterCompat( + makeAdapter({ + _testLokiAdapter: persistenceAdapter, + onPersistenceError, + onQuotaExceededError: () => { + throw new Error('quota handler failed') + }, + }), + ) + const error = quotaError() + persistenceAdapter.saveError = abortEvent(error) + await writeTask(adapter) + await waitFor(() => onPersistenceError.mock.calls.length === 1) + expect(onPersistenceError).toHaveBeenCalledWith(error) + }) +}) + +describe('LokiJSAdapter in-memory fallback', () => { + it('falls back to memory and reports it when IndexedDB is unavailable (by default)', async () => { + expect(typeof indexedDB).toBe('undefined') + const onInMemoryFallback = jest.fn() + const onSetUpError = jest.fn() + const adapter = new DatabaseAdapterCompat(makeAdapter({ onInMemoryFallback, onSetUpError })) + + await writeTask(adapter) + expect(onSetUpError).not.toHaveBeenCalled() + expect(onInMemoryFallback).toHaveBeenCalledTimes(1) + expect(onInMemoryFallback.mock.calls[0][0].message).toMatch('IndexedDB is not available') + }) + + it('reports the fallback for each adapter, even if they share options', async () => { + const onInMemoryFallback = jest.fn() + const options = { + dbName: `test${Math.random()}`, + schema: testSchema, + useWebWorker: false, + useIncrementalIndexedDB: true, + onInMemoryFallback, + } + // not using makeAdapter(), so that both adapters get the very same options object + const adapter1 = new LokiJSAdapter(options) + const adapter2 = new LokiJSAdapter(options) + adapters.push(adapter1, adapter2) + await writeTask(new DatabaseAdapterCompat(adapter1)) + await writeTask(new DatabaseAdapterCompat(adapter2)) + expect(onInMemoryFallback).toHaveBeenCalledTimes(2) + }) + + it('fails setup when IndexedDB is unavailable and allowInMemoryFallback is false', async () => { + const onInMemoryFallback = jest.fn() + const error = await new Promise((resolve) => { + makeAdapter({ allowInMemoryFallback: false, onInMemoryFallback, onSetUpError: resolve }) + }) + expect(error.message).toMatch('IndexedDB is not available') + expect(onInMemoryFallback).not.toHaveBeenCalled() + }) + + it('sets up in memory even if onInMemoryFallback throws', async () => { + const onSetUpError = jest.fn() + const adapter = new DatabaseAdapterCompat( + makeAdapter({ + onSetUpError, + onInMemoryFallback: () => { + throw new Error('fallback handler failed') + }, + }), + ) + await writeTask(adapter) + expect(onSetUpError).not.toHaveBeenCalled() + }) + + it('handles IndexedDB throwing synchronously when opening', async () => { + const idbError = new Error('The operation is insecure.') + idbError.name = 'SecurityError' + global.indexedDB = { + open: () => { + throw idbError + }, + } + + const onInMemoryFallback = jest.fn() + await writeTask(new DatabaseAdapterCompat(makeAdapter({ onInMemoryFallback }))) + expect(onInMemoryFallback).toHaveBeenCalledTimes(1) + expect(onInMemoryFallback.mock.calls[0][0].cause).toBe(idbError) + + const setUpError = await new Promise((resolve) => { + makeAdapter({ allowInMemoryFallback: false, onSetUpError: resolve }) + }) + expect(setUpError.cause).toBe(idbError) + }) + + it('does not hang setup if onQuotaExceededError throws', async () => { + const idbError = quotaError() + global.indexedDB = { + open: () => { + const request = {} + setTimeout(() => request.onerror({ target: { error: idbError } }), 0) + return request + }, + } + const onInMemoryFallback = jest.fn() + await writeTask( + new DatabaseAdapterCompat( + makeAdapter({ + onInMemoryFallback, + onQuotaExceededError: () => { + throw new Error('quota handler failed') + }, + }), + ), + ) + expect(onInMemoryFallback).toHaveBeenCalledTimes(1) + }) + + it('reports why IndexedDB failed to open', async () => { + const idbError = quotaError() + global.indexedDB = { + open: () => { + const request = {} + setTimeout(() => request.onerror({ target: { error: idbError } }), 0) + return request + }, + } + const onQuotaExceededError = jest.fn() + + const onInMemoryFallback = jest.fn() + const adapter = new DatabaseAdapterCompat( + makeAdapter({ onInMemoryFallback, onQuotaExceededError }), + ) + await writeTask(adapter) + expect(onQuotaExceededError).toHaveBeenCalledWith(idbError) + expect(onInMemoryFallback).toHaveBeenCalledTimes(1) + expect(onInMemoryFallback.mock.calls[0][0].cause).toBe(idbError) + + const setUpError = await new Promise((resolve) => { + makeAdapter({ allowInMemoryFallback: false, onSetUpError: resolve }) + }) + expect(setUpError.cause).toBe(idbError) + }) +}) diff --git a/src/adapters/lokijs/worker/DatabaseDriver.js b/src/adapters/lokijs/worker/DatabaseDriver.js index f9731e575..0cb38929e 100644 --- a/src/adapters/lokijs/worker/DatabaseDriver.js +++ b/src/adapters/lokijs/worker/DatabaseDriver.js @@ -51,7 +51,9 @@ export default class DatabaseDriver { constructor(options: LokiAdapterOptions): void { const { schema, migrations } = options - this.options = options + // copied, so that this object identifies this driver, even if the app reuses its options object + // for another adapter (see lokiExtensions) + this.options = { ...options } this.schema = schema this.migrations = migrations } @@ -219,6 +221,8 @@ export default class DatabaseDriver { } unsafeExecute(operations: UnsafeExecuteOperations): void { + // raw access could change data that can no longer be saved + this._assertNotBroken() if (process.env.NODE_ENV !== 'production') { invariant( operations && @@ -233,6 +237,9 @@ export default class DatabaseDriver { } async unsafeResetDatabase(): Promise { + // A broken driver must not delete and re-create the database (e.g. if it was deleted in + // another tab, that would bring it back with this driver's autosave) + this._assertNotBroken() await deleteDatabase(this.loki) this.cachedRecords.clear() @@ -284,7 +291,7 @@ export default class DatabaseDriver { async _openDatabase(): Promise { logger.log('[Loki] Initializing IndexedDB') - this.loki = await newLoki(this.options) + this.loki = await newLoki(this.options, (error) => this._breakDriver(error)) logger.log('[Loki] Database loaded') } @@ -464,18 +471,28 @@ export default class DatabaseDriver { ) throw error } + this._breakDriver(error) + + // Rethrow error + throw error + } + + // Puts driver in a broken state: further changes are rejected and Loki won't save anymore. + // Used directly (regardless of experimentalAllowsFatalError) when the database can no longer be + // safely persisted, e.g. because it was deleted in another browser tab + _breakDriver(error: Error): void { + if (this._isBroken) { + return + } // Stop further mutations this._isBroken = true - // Disable Loki autosave - lokiFatalError(this.loki) + // Disable Loki autosave (Loki might not be assigned yet if this happens while it's loading) + this.loki && lokiFatalError(this.loki) // Notify handler logger.error('DatabaseDriver is broken. App must be reloaded before continuing.') const handler = this.options._onFatalError handler && handler(error) - - // Rethrow error - throw error } } diff --git a/src/adapters/lokijs/worker/lokiExtensions.js b/src/adapters/lokijs/worker/lokiExtensions.js index 4fafa5f7d..cf7848d11 100644 --- a/src/adapters/lokijs/worker/lokiExtensions.js +++ b/src/adapters/lokijs/worker/lokiExtensions.js @@ -6,20 +6,41 @@ import logger from '../../../utils/common/logger' import type { LokiAdapterOptions } from '../index' import type { Loki } from '../type' -const isIDBAvailable = (onQuotaExceededError: ?(error: Error) => void) => { +// App callbacks are called from Loki timers, IndexedDB event handlers and adapter setup, and must +// not change what happens next (or prevent other callbacks), so errors thrown by them are logged +function callSafely(callback: ?(T) => void, arg: T): void { + try { + callback && callback(arg) + } catch (error) { + logger.error('[Loki] Error thrown by a LokiJSAdapter callback', error) + } +} + +// Resolves to `null` if IndexedDB can be used, or to an Error explaining why it can't +const checkIDB = (onQuotaExceededError: ?(error: Error) => void): Promise => { return new Promise((resolve) => { - // $FlowFixMe - if (typeof indexedDB === 'undefined') { - resolve(false) - } + let checkRequest: IDBOpenDBRequest + try { + // NOTE: Even accessing `indexedDB` can throw (e.g. in a sandboxed iframe) + // $FlowFixMe + if (typeof indexedDB === 'undefined') { + resolve(new Error('IndexedDB is not available in this environment')) + return + } - // in Firefox private mode, IDB will be available, but will fail to open - // $FlowFixMe - const checkRequest: IDBOpenDBRequest = indexedDB.open('WatermelonIDBChecker') + // in Firefox private mode, IDB will be available, but will fail to open + // $FlowFixMe + checkRequest = indexedDB.open('WatermelonIDBChecker') + } catch (error) { + // e.g. SecurityError when storage access is denied + logger.error('[Loki] IndexedDB checker failed to open', error) + resolve(error) + return + } checkRequest.onsuccess = (e) => { const db: IDBDatabase = e.target.result db.close() - resolve(true) + resolve(null) } checkRequest.onerror = (event) => { const error: ?Error = event?.target?.error @@ -27,15 +48,15 @@ const isIDBAvailable = (onQuotaExceededError: ?(error: Error) => void) => { // DOMException: "A mutation operation was attempted on a database that did not allow mutations." // code: 11, name: InvalidStateError logger.error( - '[Loki] IndexedDB checker failed to open. Most likely, user is in Private Mode. It could also be a quota exceeded error. Will fall back to in-memory database.', + '[Loki] IndexedDB checker failed to open. Most likely, user is in Private Mode. It could also be a quota exceeded error.', event, error, ) if (error && error.name === 'QuotaExceededError') { logger.log('[Loki] Looks like disk quota was exceeded: ', error) - onQuotaExceededError && onQuotaExceededError(error) + callSafely(onQuotaExceededError, error) } - resolve(false) + resolve(error || new Error('IndexedDB failed to open')) } checkRequest.onblocked = () => { logger.error('IndexedDB checker call is blocked') @@ -43,6 +64,11 @@ const isIDBAvailable = (onQuotaExceededError: ?(error: Error) => void) => { }) } +// Setting up an empty database opens it twice (see DatabaseDriver._migrateIfNeeded), so we keep +// track of what was reported, so that `onInMemoryFallback` is only called once per adapter +// (DatabaseDriver copies options, so each adapter has its own options object) +const reportedInMemoryFallbacks: WeakSet = new WeakSet() + async function getLokiAdapter(options: LokiAdapterOptions): mixed { const { useIncrementalIndexedDB, @@ -50,16 +76,22 @@ async function getLokiAdapter(options: LokiAdapterOptions): mixed { onQuotaExceededError, dbName, extraIncrementalIDBOptions = {}, + allowInMemoryFallback = true, + onInMemoryFallback, } = options if (adapter) { return adapter - } else if (await isIDBAvailable(onQuotaExceededError)) { + } + + const idbError = await checkIDB(onQuotaExceededError) + if (!idbError) { if (useIncrementalIndexedDB) { const IncrementalIDBAdapter = options._betaLoki ? require('lokijs/src/incremental-indexeddb-adapter') : require('lokijs/src/incremental-indexeddb-adapter') + // NOTE: options are copied, because newLoki() installs its own `onversionchange` handler // $FlowFixMe - return new IncrementalIDBAdapter(extraIncrementalIDBOptions) + return new IncrementalIDBAdapter({ ...extraIncrementalIDBOptions }) } const LokiIndexedAdapter = require('lokijs/src/loki-indexed-adapter') return new LokiIndexedAdapter(dbName) @@ -67,30 +99,146 @@ async function getLokiAdapter(options: LokiAdapterOptions): mixed { // if IDB is unavailable (that happens in private mode), fall back to memory adapter // we could also fall back to localstorage adapter, but it will fail in all but the smallest dbs + const reason = new Error( + `[Loki] IndexedDB cannot be used (${idbError.name}: ${idbError.message}), so the database can only be kept in memory and will be lost when the page is closed`, + ) + // $FlowFixMe + reason.cause = idbError + if (!allowInMemoryFallback) { + logger.error(`${reason.message}. Refusing to set up, because allowInMemoryFallback is false`) + throw reason + } + if (!reportedInMemoryFallbacks.has(options)) { + reportedInMemoryFallbacks.add(options) + logger.error(`${reason.message}. Falling back to in-memory database`) + callSafely(onInMemoryFallback, reason) + } + const { LokiMemoryAdapter } = options._betaLoki ? require('lokijs') : require('lokijs') return new LokiMemoryAdapter() } -export async function newLoki(options: LokiAdapterOptions): Loki { +// Loki reports save errors as whatever the persistence adapter passes: an Error, or, in case of +// IncrementalIndexedDBAdapter, sometimes the raw IndexedDB `error`/`abort` Event +function normalizePersistenceError(rawError: any): Error { + if (rawError instanceof Error) { + return rawError + } + const targetError = rawError && rawError.target && rawError.target.error + if (targetError) { + return targetError + } + if (rawError && rawError.type) { + return new Error(`IndexedDB transaction failed (${rawError.type} event)`) + } + return new Error(`Unknown error while saving database: ${String(rawError)}`) +} + +function createPersistenceErrorReporter(options: LokiAdapterOptions): { + onSaveResult: (rawError: any) => void, + reportFatal: (error: Error) => void, +} { + const { onPersistenceError, onQuotaExceededError } = options + // We only report the first failure until a save succeeds again, since autosave retries + // a failing save every `autosaveInterval` + let isFailing = false + let isDead = false + + const report = (error: Error) => { + if (error && error.name === 'QuotaExceededError') { + callSafely(onQuotaExceededError, error) + } + callSafely(onPersistenceError, error) + } + + return { + onSaveResult: (rawError) => { + if (isDead) { + return + } + if (!rawError) { + if (isFailing) { + logger.log('[Loki] Database saved successfully after previous failures') + isFailing = false + } + return + } + if (isFailing) { + return + } + isFailing = true + const error = normalizePersistenceError(rawError) + logger.error( + '[Loki] Failed to save database. Recent changes are only in memory and will be lost if the page is closed. Will keep retrying.', + error, + ) + report(error) + }, + reportFatal: (error) => { + if (isDead) { + return + } + isDead = true + isFailing = true + logger.error(error.message) + report(error) + }, + } +} + +// `onPersistenceLost` is called when the database can no longer be saved and the caller should +// stop accepting changes (currently: IndexedDB was deleted or upgraded by another connection) +export async function newLoki( + options: LokiAdapterOptions, + onPersistenceLost?: (error: Error) => void, +): Loki { const { extraLokiOptions = {} } = options const LokiDb = options._betaLoki ? require('lokijs') : require('lokijs') + const reporter = createPersistenceErrorReporter(options) + const adapter: any = await getLokiAdapter(options) + + let loki: ?Loki = null + + // When another connection deletes (or upgrades) the IndexedDB database - most likely, the user + // logged out in another tab - IncrementalIDBAdapter closes its connection. If we kept going, + // the next autosave would silently re-create the database from this tab's memory, bringing + // the deleted data back. Instead, we stop saving and report it. + if (adapter && adapter.mode === 'incremental' && adapter.options) { + const userOnVersionChange = options.extraIncrementalIDBOptions?.onversionchange + adapter.options.onversionchange = (event: any) => { + const error = new Error( + '[Loki] IndexedDB database was deleted or upgraded by another connection (most likely in another browser tab). To avoid re-creating it from memory, this database will not be saved anymore. App must be reloaded.', + ) + error.name = 'VersionChangeError' + // Stop saving and accepting changes before notifying app callbacks, so that they can't make + // changes anymore, and so that a throwing callback can't prevent it + loki && lokiFatalError(loki) + callSafely(onPersistenceLost, error) + reporter.reportFatal(error) + // $FlowFixMe + callSafely(userOnVersionChange, event) + } + } + // $FlowFixMe - const loki: Loki = new LokiDb(options.dbName, { - adapter: await getLokiAdapter(options), + loki = new LokiDb(options.dbName, { + adapter, autosave: true, autosaveInterval: 500, + autosaveCallback: reporter.onSaveResult, verbose: true, ...extraLokiOptions, }) + const _loki: Loki = loki // force load database now await new Promise((resolve, reject) => { - loki.loadDatabase({}, (error) => { + _loki.loadDatabase({}, (error) => { error ? reject(error) : resolve() }) }) - return loki + return _loki } export async function deleteDatabase(loki: Loki): Promise {