diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0147f99 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +packages/replicate/assets/wa-sqlite/** -text -whitespace linguist-vendored diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4752219..5db15d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,16 +6,23 @@ on: pull_request: branches: [main] +permissions: + contents: read + jobs: ci: name: Lint & Type Check runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 24 - name: Setup Bun - uses: oven-sh/setup-bun@v2 + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - name: Install dependencies run: bun install --frozen-lockfile @@ -23,8 +30,23 @@ jobs: - name: Build packages run: bun run build:packages + - name: Test replication contracts + run: bun run test + + - name: Verify SQLite assets + run: bun run --filter @trestleinc/replicate test:assets + + - name: Install Chromium + working-directory: packages/replicate + run: bunx playwright install --with-deps chromium + + - name: Test offline browser restarts + run: bun run --filter @trestleinc/replicate test:browser + - name: Lint run: bun run lint - name: Type check + env: + PUBLIC_CONVEX_URL: https://example.convex.cloud run: bun run check diff --git a/.gitignore b/.gitignore index 449e9c0..835f9f5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ # Dist node_modules dist/ +# Versioned upstream runtime distribution (not this repository's build output). +!packages/replicate/assets/wa-sqlite/dist/ +!packages/replicate/assets/wa-sqlite/dist/* packages/*/dist apps/*/dist apps/*/.env diff --git a/.prettierignore b/.prettierignore index 0cd4ddb..9a6db35 100644 --- a/.prettierignore +++ b/.prettierignore @@ -5,3 +5,5 @@ bun.lock **/dist/ **/build/ **/_generated/ + +packages/replicate/assets/ diff --git a/AGENTS.md b/AGENTS.md index 1dfbb3c..db63f7e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,14 +62,14 @@ schema.prose(); // Validator for prose fields ```typescript collection.create(); // Main entry point - create lazy-initialized collections -persistence.web.sqlite(); // Browser wa-sqlite Web Worker + OPFSCoopSyncVFS +persistence.web.sqlite.create(); // Browser wa-sqlite Web Worker + IDBBatchAtomicVFS (IndexedDB) persistence.web.sqlite.once(); // SQLite singleton mode (shared across collections) -persistence.web.encrypted(); // Browser encrypted storage (WebAuthn PRF) -persistence.native.sqlite(); // React Native SQLite persistence (op-sqlite) -persistence.native.encrypted(); // React Native encrypted storage (not yet implemented) -persistence.memory(); // In-memory persistence (testing) -persistence.custom(); // Custom storage adapter -schema.prose(); // Zod schema for prose fields +persistence.web.encryption.create(); // Browser encrypted storage (WebAuthn PRF) +persistence.web.encryption.manager(); // Optional encryption lifecycle manager +persistence.native.sqlite.create(); // React Native SQLite persistence (op-sqlite) +persistence.native.encryption.create(); // React Native encrypted storage (not yet implemented) +persistence.memory.create(); // In-memory persistence (testing) +persistence.custom.create(); // Custom storage adapter schema.prose.extract(); // Extract plain text from prose JSON schema.prose.empty(); // Create empty prose value ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index e2dc816..dcc3485 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,28 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Manual connection gating, read-only collections, captured application request arguments, and awaitable connect/disconnect/reset lifecycle. +- Versioned, checksummed self-hostable SQLite runtime assets and offline browser process-restart tests. +- Bounded, resumable `migrateSnapshots` component migration for legacy shared snapshot sequence positions. + +### Fixed + +- Persisted rows become ready without waiting for network recovery. +- Deletion and recovery absence drain/delete durable document state; failed writes cannot advance cursors or acknowledge undurable vectors. +- Raw updates with missing Yjs dependencies survive restart; stale network generations cannot repopulate reset replicas. +- Disconnect/reset followed by collection cleanup stops each Convex subscription once and ignores queued callbacks after unsubscribe. +- Full-row updates remove omitted optional fields, SQLite document clearing is atomic, and browser worker shutdown drains accepted work while rejecting new work after closing starts. +- Snapshot/delta discovery works after compaction, including scheduled compaction and writes between small pages. +- Rejected deletes persist a marker-removal rollback before republishing and retain private full-state repair metadata across restart until the next explicit update or delete succeeds. + +### Upgrade notes + +Existing v2 component collections must complete snapshot migration before streaming. See [Durable offline replicas](docs/offline-replicas.md) for mixed-client support, pending writable creations, third-party persistence capabilities, runtime provenance, and native verification limits. + ## [1.2.0-preview.0] - 2025-12-31 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 2fc5365..b74eda4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,7 +19,7 @@ Single package with exports: - `@trestleinc/replicate/client` - Client utilities (browser/React/Svelte) - `@trestleinc/replicate/server` - Server helpers (Convex functions) -- `@trestleinc/replicate/shared` - Shared types (ProseFields, XmlFragmentJSON, OperationType) +- `@trestleinc/replicate` - Shared types (ProseFields, XmlFragmentJSON, OperationType) - `@trestleinc/replicate/convex.config` - Component configuration ## Development Commands @@ -53,8 +53,8 @@ src/ │ ├── persistence/ # Swappable storage backends │ │ ├── types.ts # Persistence, KeyValueStore interfaces │ │ ├── sqlite/ # SQLite backends -│ │ │ ├── web.ts # wa-sqlite Web Worker + OPFSCoopSyncVFS -│ │ │ ├── worker.ts # Web Worker with wa-sqlite (CDN loaded) +│ │ │ ├── web.ts # wa-sqlite Web Worker + IDBBatchAtomicVFS (IndexedDB) +│ │ │ ├── worker.ts # Web Worker with configurable wa-sqlite assets │ │ │ ├── schema.ts # SQLite schema + persistence providers │ │ │ └── native.ts # op-sqlite (React Native) │ │ ├── memory.ts # Testing: in-memory @@ -163,16 +163,16 @@ Server update (via stream subscription) collection.create(); // Create lazy-initialized collection (SSR-safe) // Persistence providers (namespaced by platform) -persistence.web.sqlite(); // Browser: wa-sqlite Web Worker + OPFSCoopSyncVFS +persistence.web.sqlite.create(); // Browser: wa-sqlite Web Worker + IDBBatchAtomicVFS (IndexedDB) persistence.web.sqlite.once(); // SQLite singleton (shared across collections) -persistence.web.encrypted(); // Browser: encrypted storage with WebAuthn PRF -persistence.native.sqlite(); // React Native: op-sqlite -persistence.native.encrypted(); // React Native: encrypted storage (not yet implemented) -persistence.memory(); // Testing: in-memory (cross-platform) -persistence.custom(); // Custom storage adapter (cross-platform) +persistence.web.encryption.create(); // Browser: encrypted storage with WebAuthn PRF +persistence.web.encryption.manager(); // Optional encryption lifecycle manager +persistence.native.sqlite.create(); // React Native: op-sqlite +persistence.native.encryption.create(); // React Native: encrypted storage (not yet implemented) +persistence.memory.create(); // Testing: in-memory (cross-platform) +persistence.custom.create(); // Custom storage adapter (cross-platform) // Schema helpers (matches server API) -schema.prose(); // Zod schema for prose fields schema.prose.extract(); // Extract plain text from ProseMirror JSON schema.prose.empty(); // Create empty prose value ``` @@ -187,7 +187,7 @@ schema.table(); // Define replicated table schema schema.prose(); // Validator for ProseMirror JSON ``` -### Shared (`@trestleinc/replicate/shared`) +### Shared (`@trestleinc/replicate`) ```typescript ProseFields; // Extract prose field names from document type @@ -216,15 +216,19 @@ export const { stream, material, insert, update, remove, versions } = r({ ```typescript import { collection, persistence } from '@trestleinc/replicate/client'; +import SqliteWorker from '@trestleinc/replicate/worker?worker'; import { ConvexClient } from 'convex/browser'; +const sqlite = persistence.web.sqlite.once({ + name: 'tasks', + assetBase: '/replicate-sqlite', + worker: () => new SqliteWorker(), +}); + export const tasks = collection.create({ - persistence: async () => { - const SQL = await initSqlJs({ locateFile: (f) => `/${f}` }); - return persistence.sqlite.browser(SQL, 'tasks'); - }, + schema: taskSchema, + persistence: sqlite, config: () => ({ - schema: taskSchema, convexClient: new ConvexClient(import.meta.env.VITE_CONVEX_URL), api: api.tasks, getKey: (task) => task.id, @@ -233,10 +237,10 @@ export const tasks = collection.create({ // Initialize once during app startup (browser only) await tasks.init(); -const collection = tasks.get(); +const rows = tasks.get(); // Prose binding for rich text -const binding = await collection.utils.prose(id, 'content'); +const binding = await rows.doc(id).prose('content'); ``` ### Sync Registration (Internal) diff --git a/README.md b/README.md index 64f6119..6ec8cdc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Replicate -A local-first sync engine for [Convex](https://convex.dev). Replicate keeps a full copy of your data in client-side SQLite (via OPFS on web, op-sqlite on React Native) and syncs changes through Convex using CRDTs (Yjs). Users get instant reads, offline writes, and real-time collaboration without writing conflict resolution logic. +A local-first sync engine for [Convex](https://convex.dev). Replicate keeps a full copy of your data in client-side SQLite (via IndexedDB-backed SQLite on web, op-sqlite on React Native) and syncs changes through Convex using CRDTs (Yjs). Users get instant reads, offline writes, and real-time collaboration without writing conflict resolution logic. + +For gated synchronization, read-only replicas, durable reset, browser precaching, and existing-component upgrades, see [Durable offline replicas](docs/offline-replicas.md). Existing v2 snapshots require the documented migration before streaming. ## Architecture @@ -36,7 +38,7 @@ graph TB App[Application] Collection[TanStack DB Collection] Yjs[Yjs CRDT Documents] - SQLite[SQLite / OPFS] + SQLite[SQLite / IndexedDB] end subgraph Server @@ -79,7 +81,7 @@ Install the Replicate component in your Convex project: ```typescript // convex/convex.config.ts import { defineApp } from 'convex/server'; -import replicate from '@trestleinc/replicate/component'; +import replicate from '@trestleinc/replicate/convex.config'; const app = defineApp(); app.use(replicate); @@ -190,6 +192,7 @@ import type { Infer } from 'convex/values'; const sqlite = persistence.web.sqlite.once({ name: 'myapp', + assetBase: '/replicate-sqlite', worker: async () => { const { default: SqliteWorker } = await import('@trestleinc/replicate/worker?worker'); return new SqliteWorker(); @@ -209,6 +212,9 @@ export const tasks = collection.create({ export type Task = Infer; ``` +For offline startup, copy the packaged runtime with `replicate-sqlite-assets public/replicate-sqlite` +and precache it with the application shell. See [Durable offline replicas](docs/offline-replicas.md#browser-assets-and-offline-startup). + ### Step 4: Initialize and Use Initialize the collection with optional server-seeded data, then query it like any TanStack DB collection: @@ -360,19 +366,20 @@ const empty = schema.prose.empty(); Replicate ships with several persistence backends: -| Provider | Platform | Description | -| -------------------------------------------- | ------------ | ---------------------------------------------- | -| `persistence.web.sqlite.create(options)` | Browser | wa-sqlite in a Web Worker with OPFS storage | -| `persistence.web.sqlite.once(options)` | Browser | Same as above, but returns a singleton factory | -| `persistence.native.sqlite.create(db, name)` | React Native | op-sqlite integration | -| `persistence.memory.create()` | Any | In-memory (for testing) | -| `persistence.custom.create(adapter)` | Any | Bring your own `StorageAdapter` | +| Provider | Platform | Description | +| -------------------------------------------- | ------------ | ------------------------------------------------ | +| `persistence.web.sqlite.create(options)` | Browser | wa-sqlite in a Web Worker with IndexedDB storage | +| `persistence.web.sqlite.once(options)` | Browser | Same as above, but returns a singleton factory | +| `persistence.native.sqlite.create(db, name)` | React Native | op-sqlite integration | +| `persistence.memory.create()` | Any | In-memory (for testing) | +| `persistence.custom.create(adapter)` | Any | Bring your own `StorageAdapter` | **Web SQLite options:** ```typescript interface WebSqliteOptions { - name: string; // Database name in OPFS + name: string; // Database name in IndexedDB + assetBase?: string; // Base URL for the compatible wa-sqlite runtime graph worker: Worker | (() => Worker | Promise); } ``` @@ -637,37 +644,34 @@ await configure({ ### Encrypted Storage -Encrypt the local SQLite database using WebAuthn PRF (Passkey-derived keys): +Encrypt persisted document payloads and application key-value payloads using WebAuthn PRF (Passkey-derived keys). Storage metadata, including key names and the document-ID discovery index, remains visible to the backing persistence provider: ```typescript import { persistence } from '@trestleinc/replicate/client'; // Check if the browser supports PRF const supported = await persistence.web.encryption.webauthn.supported(); +if (!supported) throw new Error('WebAuthn PRF is required by this configuration'); -// Create an encryption manager (handles key lifecycle) -const manager = persistence.web.encryption.manager({ - rpId: 'example.com', - rpName: 'My App', - onStateChange: (state) => { - /* update UI */ +const storage = await persistence.web.sqlite.create({ + name: 'myapp-encrypted', + assetBase: '/replicate-sqlite', + worker: async () => { + const { default: W } = await import('@trestleinc/replicate/worker?worker'); + return new W(); }, }); -// Create encrypted persistence -const encrypted = () => - persistence.web.encryption.create({ - manager, - name: 'myapp-encrypted', - worker: async () => { - const { default: W } = await import('@trestleinc/replicate/worker?worker'); - return new W(); - }, - }); +const encrypted = await persistence.web.encryption.create({ + storage, + user: currentUser.id, + unlock: { webauthn: true }, +}); +await encrypted.unlock(); const tasks = collection.create({ schema: taskSchema, - persistence: encrypted, + persistence: async () => encrypted, config: () => ({ /* ... */ }), @@ -835,6 +839,9 @@ function create(options: CreateCollectionOptions): LazyColl interface CreateCollectionOptions { schema: VersionedSchema; persistence: () => Promise; + networkMode?: 'auto' | 'manual'; // Defaults to auto + readOnly?: boolean; // Defaults to false + requestArgs?: () => Record; // Captured for each connection config: () => { convexClient: ConvexClient; api: ConvexCollectionApi; @@ -847,6 +854,10 @@ interface CreateCollectionOptions { } ``` +`tasks.get()` returns the TanStack collection with `connect()`, `disconnect()`, and `reset()` +methods. In manual mode, call and await `connect()` after credentials are available. Await +`disconnect()` before changing account context and `reset()` when clearing the durable replica. + #### `LazyCollection` ```typescript @@ -994,11 +1005,11 @@ interface UserIdentity { } ``` -### Shared Types (`@trestleinc/replicate/shared`) +### Shared Types (`@trestleinc/replicate`) ```typescript -import type { ProseValue, Duration, CompactionConfig } from '@trestleinc/replicate/shared'; -import { parseDuration } from '@trestleinc/replicate/shared'; +import type { ProseValue, Duration, CompactionConfig } from '@trestleinc/replicate'; +import { parseDuration } from '@trestleinc/replicate'; // Duration strings type Duration = `${number}${'m' | 'h' | 'd'}`; @@ -1064,7 +1075,7 @@ A full-featured offline-first issue tracker demonstrating real-world usage. **Features demonstrated:** -- Offline-first with wa-sqlite persistence (SQLite in OPFS) +- Offline-first with wa-sqlite persistence (SQLite in IndexedDB) - Rich text editing with TipTap + Yjs collaboration - PWA with custom service worker for offline navigation - Real-time sync across devices diff --git a/apps/sveltekit/package.json b/apps/sveltekit/package.json index 9040ebd..5475bab 100644 --- a/apps/sveltekit/package.json +++ b/apps/sveltekit/package.json @@ -41,7 +41,6 @@ "convex": "catalog:", "convex-svelte": "catalog:", "tailwind-merge": "catalog:", - "wa-sqlite": "catalog:", "lib0": "catalog:", "y-protocols": "catalog:", "yjs": "catalog:", diff --git a/bun.lock b/bun.lock index b062e3f..be7968a 100644 --- a/bun.lock +++ b/bun.lock @@ -108,7 +108,6 @@ "convex-svelte": "catalog:", "lib0": "catalog:", "tailwind-merge": "catalog:", - "wa-sqlite": "catalog:", "y-protocols": "catalog:", "yjs": "catalog:", "zod": "catalog:", @@ -213,9 +212,13 @@ "packages/replicate": { "name": "@trestleinc/replicate", "version": "2.0.0-preview.0", + "bin": { + "replicate-sqlite-assets": "./scripts/copy-sqlite-assets.mjs", + }, "dependencies": { "@logtape/logtape": "catalog:", "@logtape/pretty": "catalog:", + "wa-sqlite": "https://codeload.github.com/rhashimoto/wa-sqlite/tar.gz/07ad48cf2f682d279f9cd69818cc1d39b2ccda86", }, "devDependencies": { "@convex-dev/eslint-plugin": "catalog:", @@ -243,6 +246,7 @@ "tsdown": "0.20.0-beta.3", "typescript": "catalog:", "typescript-eslint": "catalog:", + "vite": "catalog:", "vitest": "catalog:", }, "peerDependencies": { @@ -250,13 +254,11 @@ "@tanstack/db": "^0.5.20", "convex": "^1.31.4", "lib0": "^0.2.0", - "wa-sqlite": "1.0.0", "y-protocols": "1.0.7", "yjs": "13.6.29", }, "optionalPeers": [ "@op-engineering/op-sqlite", - "wa-sqlite", ], }, "workers/wa-sqlite-cdn": { @@ -349,7 +351,6 @@ "typescript-eslint": "^8.52.0", "vite": "^7.3.1", "vitest": "^4.0.17", - "wa-sqlite": "^1.0.0", "wrangler": "^4.56.0", "y-protocols": "^1.0.7", "yjs": "^13.6.29", @@ -3478,7 +3479,7 @@ "w3c-keyname": ["w3c-keyname@2.2.8", "", {}, "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ=="], - "wa-sqlite": ["wa-sqlite@1.0.0", "", {}, "sha512-Kyybo5/BaJp76z7gDWGk2J6Hthl4NIPsE+swgraEjy3IY6r5zIR02wAs1OJH4XtJp1y3puj3Onp5eMGS0z7nUA=="], + "wa-sqlite": ["wa-sqlite@https://codeload.github.com/rhashimoto/wa-sqlite/tar.gz/07ad48cf2f682d279f9cd69818cc1d39b2ccda86", {}], "wa-sqlite-cdn": ["wa-sqlite-cdn@workspace:workers/wa-sqlite-cdn"], diff --git a/docs/CDN.md b/docs/CDN.md index 9a340bb..30a6f58 100644 --- a/docs/CDN.md +++ b/docs/CDN.md @@ -1,319 +1,166 @@ -# Self-Hosting wa-sqlite on Cloudflare +# Hosting the browser SQLite runtime -This guide explains how to set up your own CDN for wa-sqlite to maximize performance and reliability for your users. +Replicate's browser persistence uses wa-sqlite with `IDBBatchAtomicVFS` (IndexedDB). Replicate +installs an immutable archive of the original repository at commit +`07ad48cf2f682d279f9cd69818cc1d39b2ccda86`. Its asset CLI copies the compatible +JavaScript/WASM graph, verifies fixed checksums, and retains the upstream license, +acknowledgments, and provenance note. Serve that graph from your application or an immutable CDN +path. Do not assemble files from a moving branch or substitute npm `wa-sqlite@1.0.0`, an +unofficial publication with a different VFS API. -## Why Self-Host? +## Copy and verify the packaged graph -| Benefit | Description | -| ---------------- | ----------------------------------------------------------- | -| **Performance** | Cloudflare's edge network (300+ locations) vs jsdelivr | -| **Reliability** | No dependency on third-party CDN uptime | -| **Control** | Pin versions, custom cache headers, monitoring | -| **Native users** | wa-sqlite only loaded for web, not bundled for React Native | - -## Current Architecture - -``` -┌─────────────────────────────────────────────────────────────┐ -│ Web App │ -│ └── Web Worker │ -│ └── Dynamic imports from CDN │ -│ ├── wa-sqlite-async.mjs (~15KB) │ -│ ├── IDBBatchAtomicVFS.js (~12KB) │ -│ ├── sqlite-api.js (~20KB) │ -│ └── wa-sqlite-async.wasm (~400KB) │ -└─────────────────────────────────────────────────────────────┘ -``` - -> **Note:** We use `IDBBatchAtomicVFS` (IndexedDB-based) instead of OPFS adapters for stability. - -## Setup Options - -### Option A: Cloudflare R2 (Recommended) - -Best for static file hosting with global edge caching. - -#### 1. Create R2 Bucket - -```bash -# Using Wrangler CLI -wrangler r2 bucket create wa-sqlite-cdn -``` - -#### 2. Upload wa-sqlite Files +Run the package CLI during the application build: ```bash -# Clone wa-sqlite and upload dist + examples -git clone https://github.com/rhashimoto/wa-sqlite.git -cd wa-sqlite - -# Upload to R2 - core async build -wrangler r2 object put wa-sqlite-cdn/v1.0.0/dist/wa-sqlite-async.mjs --file=dist/wa-sqlite-async.mjs -wrangler r2 object put wa-sqlite-cdn/v1.0.0/dist/wa-sqlite-async.wasm --file=dist/wa-sqlite-async.wasm -wrangler r2 object put wa-sqlite-cdn/v1.0.0/src/sqlite-api.js --file=src/sqlite-api.js - -# Upload IDBBatchAtomicVFS and its dependencies -wrangler r2 object put wa-sqlite-cdn/v1.0.0/src/examples/IDBBatchAtomicVFS.js --file=src/examples/IDBBatchAtomicVFS.js -wrangler r2 object put wa-sqlite-cdn/v1.0.0/src/FacadeVFS.js --file=src/FacadeVFS.js -wrangler r2 object put wa-sqlite-cdn/v1.0.0/src/VFS.js --file=src/VFS.js -wrangler r2 object put wa-sqlite-cdn/v1.0.0/src/WebLocksMixin.js --file=src/WebLocksMixin.js -wrangler r2 object put wa-sqlite-cdn/v1.0.0/src/sqlite-constants.js --file=src/sqlite-constants.js +npm exec -- replicate-sqlite-assets public/replicate-sqlite ``` -#### 3. Configure Public Access - -Create a Worker to serve R2 with proper headers: - -```typescript -// workers/cdn.ts -export default { - async fetch(request: Request, env: Env): Promise { - const url = new URL(request.url); - const key = url.pathname.slice(1); // Remove leading slash - - const object = await env.WA_SQLITE_BUCKET.get(key); - - if (!object) { - return new Response('Not Found', { status: 404 }); - } - - const headers = new Headers(); - - // Set content type - if (key.endsWith('.wasm')) { - headers.set('Content-Type', 'application/wasm'); - } else if (key.endsWith('.js') || key.endsWith('.mjs')) { - headers.set('Content-Type', 'application/javascript'); - } - - // Cache headers - immutable for versioned paths - headers.set('Cache-Control', 'public, max-age=31536000, immutable'); - headers.set('Access-Control-Allow-Origin', '*'); - headers.set('Access-Control-Allow-Methods', 'GET, HEAD, OPTIONS'); +The command verifies every source file against `manifest.json` before copying. The destination +contains the complete required layout: - return new Response(object.body, { headers }); - }, -}; +```text +replicate-sqlite/ +├── dist/ +│ ├── wa-sqlite-async.mjs +│ └── wa-sqlite-async.wasm +├── src/ +│ ├── examples/IDBBatchAtomicVFS.js +│ ├── FacadeVFS.js +│ ├── VFS.js +│ ├── WebLocksMixin.js +│ ├── sqlite-api.js +│ └── sqlite-constants.js +├── manifest.json +├── LICENSE +└── ACKNOWLEDGMENTS.md ``` -#### 4. Deploy Worker - -```toml -# wrangler.toml -name = "wa-sqlite-cdn" -main = "workers/cdn.ts" -compatibility_date = "2024-01-01" - -[[r2_buckets]] -binding = "WA_SQLITE_BUCKET" -bucket_name = "wa-sqlite-cdn" - -[routes] -pattern = "wa-sqlite.trestle.inc/*" -``` +The copied directory also contains `PROVENANCE.md`. Keep the relative `dist/` and `src/` paths +intact and retain all three notice files in every hosted distribution. To verify the installed +package without copying it, run: ```bash -wrangler deploy +npm exec -- replicate-sqlite-assets --verify ``` -### Option B: Cloudflare Workers (Proxy + Cache) - -For dynamic proxying with automatic updates: - -```typescript -// workers/wa-sqlite-proxy.ts -const UPSTREAM = 'https://cdn.jsdelivr.net/gh/rhashimoto/wa-sqlite@master'; -const CACHE_TTL = 86400 * 7; // 7 days +`manifest.json` records the exact archive URL and integrity, source revision, and SHA-256 digest +of each runtime and notice file. The copy command reads the installed dependency during the +application build. When `assetBase` points to the copied directory, the SQLite worker loads that +directory instead of GitHub or the default CDN. Service-worker and preload URLs are configured +separately and must point to the same local graph. The provenance details and reproducibility +boundary are documented in [`offline-replicas.md`](offline-replicas.md#browser-assets-and-offline-startup). -export default { - async fetch(request: Request): Promise { - const url = new URL(request.url); - const upstreamUrl = `${UPSTREAM}${url.pathname}`; +## Configure Replicate - // Check cache first - const cache = caches.default; - let response = await cache.match(request); - - if (!response) { - response = await fetch(upstreamUrl); - - // Clone and add cache headers - response = new Response(response.body, response); - response.headers.set('Cache-Control', `public, max-age=${CACHE_TTL}`); - response.headers.set('Access-Control-Allow-Origin', '*'); - - // Store in cache - await cache.put(request, response.clone()); - } - - return response; - }, -}; -``` - -## Configuring Replicate - -### Default CDN (Your Hosted) - -Update the worker to use your CDN: - -```typescript -// src/client/persistence/sqlite/worker.ts -const CDN_BASE = 'https://wa-sqlite.trestle.inc/v1.0.0'; -``` - -### User-Configurable CDN - -Allow users to override: +Point `assetBase` at the directory containing `dist/`, `src/`, and the manifest: ```typescript -// src/client/persistence/sqlite/web.ts -export interface WebSqliteOptions { - name: string; - worker: Worker | (() => Worker | Promise); - cdnBase?: string; // Optional custom CDN -} +import { persistence } from '@trestleinc/replicate/client'; +import SqliteWorker from '@trestleinc/replicate/worker?worker'; -// In worker initialization, pass cdnBase via postMessage +const sqlite = persistence.web.sqlite.once({ + name: 'myapp', + assetBase: '/replicate-sqlite', + worker: () => new SqliteWorker(), +}); ``` -### Custom Adapter Support - -For users with completely custom setups: +`assetBase` may also be an absolute, versioned URL: ```typescript -// src/client/persistence/index.ts -export const persistence = { - sqlite: { - once: onceWebSqlitePersistence, - create: createWebSqlitePersistence, - }, - native: nativeSqlitePersistence, - memory: memoryPersistence, - custom: customPersistence, // For advanced users -}; +assetBase: 'https://assets.example.com/replicate-sqlite/2.0.0-preview.0'; ``` -## Version Management - -### Pinning Strategy +The host must serve `.wasm` as `application/wasm` and `.js`/`.mjs` as JavaScript. If the asset +origin differs from the application origin, enable CORS for the application. Versioned paths can +use long-lived immutable caching: +```http +Cache-Control: public, max-age=31536000, immutable +Access-Control-Allow-Origin: https://app.example.com ``` -wa-sqlite.trestle.inc/ -├── v1.0.0/ # Pinned version (immutable) -│ ├── dist/ -│ └── src/ -├── v1.1.0/ # New version -│ ├── dist/ -│ └── src/ -└── latest/ # Symlink to latest stable (for dev) - ├── dist/ - └── src/ -``` - -### Update Script - -```bash -#!/bin/bash -# scripts/update-wa-sqlite.sh - -VERSION=$1 -BUCKET="wa-sqlite-cdn" - -if [ -z "$VERSION" ]; then - echo "Usage: ./update-wa-sqlite.sh " - exit 1 -fi -# Clone specific tag/commit -git clone --depth 1 --branch $VERSION https://github.com/rhashimoto/wa-sqlite.git /tmp/wa-sqlite +Avoid a CDN proxy to a moving branch such as `@master`. To upgrade, update the Replicate package, +run the copy command again, upload the complete output to a new immutable path, and then change +`assetBase`. Keeping the previous path available lets already deployed clients finish loading the +runtime version they were built against. -# Upload to R2 - core async build -wrangler r2 object put $BUCKET/$VERSION/dist/wa-sqlite-async.mjs --file=/tmp/wa-sqlite/dist/wa-sqlite-async.mjs -wrangler r2 object put $BUCKET/$VERSION/dist/wa-sqlite-async.wasm --file=/tmp/wa-sqlite/dist/wa-sqlite-async.wasm -wrangler r2 object put $BUCKET/$VERSION/src/sqlite-api.js --file=/tmp/wa-sqlite/src/sqlite-api.js +## Offline startup -# Upload IDBBatchAtomicVFS and its dependencies -wrangler r2 object put $BUCKET/$VERSION/src/examples/IDBBatchAtomicVFS.js --file=/tmp/wa-sqlite/src/examples/IDBBatchAtomicVFS.js -wrangler r2 object put $BUCKET/$VERSION/src/FacadeVFS.js --file=/tmp/wa-sqlite/src/FacadeVFS.js -wrangler r2 object put $BUCKET/$VERSION/src/VFS.js --file=/tmp/wa-sqlite/src/VFS.js -wrangler r2 object put $BUCKET/$VERSION/src/WebLocksMixin.js --file=/tmp/wa-sqlite/src/WebLocksMixin.js -wrangler r2 object put $BUCKET/$VERSION/src/sqlite-constants.js --file=/tmp/wa-sqlite/src/sqlite-constants.js +An offline restart needs the application shell, bundled worker, and every runtime file named in +`manifest.json` to be available locally. Precache those URLs while online and activate the service +worker before relying on offline startup. Configuring `assetBase` controls the URLs loaded by the +SQLite worker; it does not rewrite a service worker's precache URLs. Configure both `assetBase` and +the service worker's runtime base and URL list to point at the copied graph. If an example service +worker has a hardcoded `CDN_BASE` and four-file URL list, replace both with the local base and the +complete runtime graph from `/replicate-sqlite/manifest.json`; changing `CDN_BASE` alone does not add +the remaining transitive runtime files to that list. The preload helpers accept the same base: -# Cleanup -rm -rf /tmp/wa-sqlite +```typescript +import { injectPreloadLinks } from '@trestleinc/replicate/client'; -echo "Uploaded wa-sqlite $VERSION to CDN" +injectPreloadLinks('/replicate-sqlite'); ``` -## Performance Optimizations - -### 1. Brotli Compression - -Cloudflare automatically applies Brotli for supported browsers: +Verify the production build with the network disabled after a full browser process shutdown. A +warm module in an open tab does not prove that the JavaScript/WASM graph is available offline. -- `wa-sqlite-async.wasm`: 400KB → ~150KB (62% reduction) -- JS files: ~43KB → ~12KB (72% reduction) +## Cloudflare R2 example -### 2. HTTP/2 Server Push (Optional) +Create a dedicated R2 bucket that contains only public runtime distributions; do not share this +binding with application or private objects. Upload the **entire** copied directory, including the +manifest and notice files, under the immutable prefix `replicate-sqlite/2.0.0-preview.0/`. This +Worker serves only that declared prefix and rejects other paths before reading the bucket: ```typescript -// In worker, add Link headers for push -headers.set('Link', '; rel=preload; as=fetch'); -``` +interface Env { + RUNTIME_BUCKET: R2Bucket; +} -### 3. Edge Caching Rules +const PUBLIC_RUNTIME_PREFIX = '/replicate-sqlite/2.0.0-preview.0/'; +const PUBLIC_RUNTIME_KEY_PREFIX = PUBLIC_RUNTIME_PREFIX.slice(1); -In Cloudflare Dashboard → Caching → Cache Rules: +const notFound = () => new Response('Not Found', { status: 404 }); -``` -Match: hostname eq "wa-sqlite.trestle.inc" and starts_with(http.request.uri.path, "/v") -Cache TTL: 1 year (immutable versioned content) -``` +function contentTypeFor(key: string): string { + if (key.endsWith('.wasm')) return 'application/wasm'; + if (key.endsWith('.json')) return 'application/json'; + if (key.endsWith('.md') || key.endsWith('/LICENSE')) return 'text/plain; charset=utf-8'; + if (key.endsWith('.js') || key.endsWith('.mjs')) return 'application/javascript'; + return 'application/octet-stream'; +} -## Monitoring +export default { + async fetch(request: Request, env: Env): Promise { + if (request.method !== 'GET' && request.method !== 'HEAD') { + return new Response('Method Not Allowed', { + status: 405, + headers: { Allow: 'GET, HEAD' }, + }); + } -### R2 Analytics + const url = new URL(request.url); + if (!url.pathname.startsWith(PUBLIC_RUNTIME_PREFIX)) return notFound(); -View in Cloudflare Dashboard → R2 → Analytics: + const relativePath = url.pathname.slice(PUBLIC_RUNTIME_PREFIX.length); + if (!relativePath) return notFound(); -- Request count by file -- Bandwidth usage -- Cache hit ratio + const key = PUBLIC_RUNTIME_KEY_PREFIX + relativePath; + const contentType = contentTypeFor(key); -### Custom Logging (Optional) + const object = await env.RUNTIME_BUCKET.get(key); + if (!object) return notFound(); -```typescript -// In worker -console.log( - JSON.stringify({ - timestamp: Date.now(), - path: url.pathname, - userAgent: request.headers.get('User-Agent'), - country: request.cf?.country, - }) -); + const headers = new Headers(); + headers.set('Content-Type', contentType); + headers.set('Cache-Control', 'public, max-age=31536000, immutable'); + headers.set('Access-Control-Allow-Origin', 'https://app.example.com'); + return new Response(request.method === 'HEAD' ? null : object.body, { headers }); + }, +}; ``` -View logs: `wrangler tail wa-sqlite-cdn` - -## Cost Estimate - -| Resource | Free Tier | Estimated Cost | -| ---------- | ------------ | ----------------------------- | -| R2 Storage | 10GB | ~$0.015/GB/month | -| R2 Egress | 10GB/month | Free (no egress fees!) | -| Workers | 100K req/day | Free | -| **Total** | - | **~$0-5/month** for most apps | - -## Checklist - -- [ ] Create Cloudflare account (if needed) -- [ ] Set up R2 bucket `wa-sqlite-cdn` -- [ ] Upload wa-sqlite files with version prefix -- [ ] Deploy CDN worker with CORS + cache headers -- [ ] Configure custom domain `wa-sqlite.trestle.inc` -- [ ] Update replicate worker.ts with new CDN_BASE -- [ ] Test in development and production -- [ ] Set up monitoring/alerts -- [ ] Document CDN URL for library users +Bind `RUNTIME_BUCKET` to the dedicated public-assets bucket. Do not mutate objects under the +published prefix; upload an upgrade to a new versioned prefix and update `PUBLIC_RUNTIME_PREFIX`. +The object-key prefix is derived from it. diff --git a/docs/offline-replicas.md b/docs/offline-replicas.md new file mode 100644 index 0000000..6b060ed --- /dev/null +++ b/docs/offline-replicas.md @@ -0,0 +1,165 @@ +# Durable offline replicas + +Initialize persistence and await `collection.preload()` to read local records. Local readiness does not wait for recovery, subscriptions, or credentials. Synchronization errors do not prevent offline reads. + +```ts +const sqlite = await persistence.web.sqlite.create({ + name: 'records', + assetBase: '/replicate-sqlite', + worker: async () => { + const { default: Worker } = await import('@trestleinc/replicate/worker?worker'); + return new Worker(); + }, +}); +const records = collection.create({ + schema: recordSchema, + persistence: async () => sqlite, + networkMode: 'manual', + readOnly: true, + requestArgs: () => ({ guestToken: currentGuestToken }), + config: () => ({ convexClient, api: api.records, getKey: (record) => record.id }), +}); +await records.init(); +const local = records.get(); +await local.preload(); // locally persisted rows are available, even offline +// Once the application has credentials: +await local.connect(); +``` + +`networkMode` defaults to `'auto'`. `connect()` starts background recovery and streaming; it does not wait for server synchronization. Manual mode makes no queries, subscriptions, document mutations, or presence calls before connecting. Explicit document edits in disconnected manual mode remain local and their transaction rejects; pending creations are retained for retry after connection. Existing automatic writable behavior remains the default. + +`readOnly` rejects insert/update/delete transactions and mutable prose/counter/register/set bindings. It still receives server changes and sends durable progress/presence messages. It is a client behavior option, not an authorization boundary: applications must enforce their own server write permissions. + +`requestArgs` is evaluated and cloned at each connection. Its values accompany all collection transport queries, subscriptions, mutations, recovery, and presence operations. Protocol arguments (including document, vector, seq, pagination, presence, and write fields) are reserved and rejected. The application endpoints must declare and validate these additional arguments, for example through the application's existing custom-function wrapper. Stock endpoint validators do not automatically accept extra credentials. Replicate does not interpret credentials or define an authorization policy. + +## Disconnect, account changes, and shutdown + +```ts +await local.disconnect(); // invalidates callbacks, stops subscriptions/bindings, drains local writes +await local.reset(); // disconnects, deletes this collection's durable documents and cursor +currentGuestToken = nextCredential; +await local.connect(); +// Final shutdown; a shared SQLite instance closes after all its collections: +await local.disconnect(); +await local.cleanup(); +await sqlite.close?.(); +``` + +Disconnect does not await unresolved backend requests. Their eventual results cannot affect a later connection. It retains documents; account switching should reset or use a separate database. Reset waits for earlier local persistence work, remains disconnected, and a concurrent connect waits for reset. Recreate document handles and editing bindings after reconnect. Do not retain/mutate raw Yjs fragments across reset. The database must have a single collection owner for each collection name; concurrent independent instances sharing the same persisted collection are not supported. + +Incoming raw Yjs updates are stored before application, including updates whose base has not arrived. The stream serializes pages, flushes changed documents, persists the cursor, and only then advances its subscription and reports a captured durable vector. Rows become readable once their data is durable; a canceled or failed checkpoint leaves that local data readable and eligible for replay. Failed writes keep the cursor and retry the subscription. Deletion drains writes and removes snapshots and deltas before progress advances. Recovery uses the server's existing material/view existence rules; explicit absence removes local state durably. + +Insert retries preserve a single logical document ID without repeating `onInsert`. If the row already exists, the retry uses the normal update path (including `onUpdate`) so newer offline edits in its payload are synchronized. + +Writable pending creations are marked durably before local insertion, retained across restart, and retried after connection. Legacy writable documents without a pending marker are preserved when the server has no history of that ID (`known: false`); this avoids discarding unsent pre-upgrade creations. Read-only replicas treat that absence as authoritative. A server tombstone/history with absent material remains authoritative for writable clients too. This is not a general persistent transaction outbox or a change to the existing writable retry/conflict model. + +When the server rejects a delete, Replicate restores the row with a newer Yjs transaction that removes the deletion markers, persists the rollback before republishing the row, and records private per-document repair metadata. That metadata survives restart; the next explicit successful update or delete sends full Yjs state, then clears it. It is not an automatic retry or outbox. An awaited disconnect or reset for an account switch fences stale rollback work. A crash before the failure and rollback complete retains the existing ambiguous in-flight-write limits. + +## Existing component upgrade + +This change targets the current v2 preview component schema (`deltas`, `snapshots`, `sequences`). Migration from the different npm 1.1.2 component is outside this patch; the archived migration documents describe earlier schemas and are not a verified stable-to-preview upgrade procedure. + +Existing snapshots can share numeric sequence positions, so simply indexing them would lose records at page boundaries. New compaction assigns a unique collection sequence to every snapshot publication, in the same Convex transaction as replacement and delta removal. Stream pages merge bounded snapshot/delta ranges and advance only through emitted entries. Presence retention still uses Yjs state vectors. + +After updating the component, run its `migrateSnapshots` mutation for **each existing collection**, in batches until `done` is true. Invoke it through an application **internal** administrative mutation, rather than exposing a public migration endpoint: + +```ts +const result = await ctx.runMutation(components.replicate.mutations.migrateSnapshots, { + collection: 'records', + limit: 50, +}); +// Repeat this internal operation until result.done === true. +``` + +Each batch atomically republishes at most 100 legacy snapshots above the highest existing snapshot, delta, or counter position. Completed rows have a marker; an interrupted operation resumes with the remaining rows and can safely be repeated. Writes and scheduled compaction may continue during migration. Streaming reports `Snapshot migration required` while any legacy snapshots remain; plan a synchronization maintenance window. Local reads and per-document recovery remain available. + +Old v2 clients already understand snapshot entries and numeric cursors, so they can consume the upgraded component after migration. They retain their original local durability defects until upgraded. Mixed old/new **clients** are supported; rolling back the component schema/code while retaining publication metadata is not a supported migration path. Do not run legacy compaction code against the upgraded component. + +Previously corrupt cursors above all retained server sequence evidence cannot be reconstructed. Use `reset()` for affected read-only caches after migration. Preserve/export unsent writable changes before resetting. A cursor must never be restored independently of the matching local data. + +## Custom persistence + +`persistence.custom.create(adapter)` implements the durable contract automatically. Direct third-party `PersistenceProvider` implementations must add: + +- `flush()`: await all captured writes and propagate failures; failed writes must remain retryable. +- `storeUpdate(bytes)`: durably store a raw **Yjs V2** update, even if its dependencies are missing. The collection applies it with origin `'replicate-persisted'` afterward; do not write it twice from that event. +- `clear()`: detach observers, drain initialization/writes, and delete all durable state for the document before resolving. + +The methods are optional in the TypeScript interface for source compatibility, but durable synchronization fails explicitly if a required capability is missing. The built-in SQLite, memory, custom, and encrypted providers implement them. Encrypted persistence now maintains an ID index; pre-existing encrypted documents must already be discoverable through the underlying adapter's `listDocuments()` to be recovered/indexed. + +## Browser assets and offline startup + +The browser worker uses **IDBBatchAtomicVFS (IndexedDB)**, not OPFS. Copy the versioned runtime included with the package into your application's public assets during its build: + +```sh +npm exec -- replicate-sqlite-assets public/replicate-sqlite +``` + +Repeated builds refresh the runtime files in that directory and preserve unrelated files. + +```ts +const sqlite = persistence.web.sqlite.once({ + name: 'records', + assetBase: '/replicate-sqlite', + worker: async () => { + const { default: Worker } = await import('@trestleinc/replicate/worker?worker'); + return new Worker(); + }, +}); +``` + +The packaged source manifest is `assets/wa-sqlite/manifest.json`; the copy command hosts it as `/manifest.json`. Precache the app shell, bundled worker, and **all runtime files** listed by that hosted manifest under the same base. Configure the worker's `assetBase` and the service worker's precache URLs separately; replace any hardcoded CDN URL list with the complete manifest-derived local graph. Preserve the relative `src/` and `dist/` paths and all notice files. Serve `.wasm` as `application/wasm` and JS/MJS as JavaScript. Install/activate the service worker online before the first offline restart. `preloadLinks(assetBase)` and `injectPreloadLinks(assetBase)` accept the same base. Failed WASM compilation is evicted from the cache so retry can succeed. Without `assetBase`, the existing historical CDN remains the default; copying and configuring `assetBase` is the offline self-hosted path. + +The worker uses `synchronous=FULL` and awaits SQLite/VFS close before terminating. Durability is subject to browser storage eviction, device/storage guarantees, and application cache lifecycle; offline assets are application-owned. + +### Runtime provenance + +The copied graph comes from an immutable archive of the original wa-sqlite repository at commit `07ad48cf2f682d279f9cd69818cc1d39b2ccda86`. Its package metadata reports version 1.1.2, but the dependency is the exact commit archive rather than an npm release or a similarly named Git tag. The required files are byte-identical to that commit. The shipped module reports SQLite 3.53.0, matching its Makefile. Per-file SHA-256 values are recorded in `manifest.json`. + +The codeload URL contains the full commit hash, and `manifest.json` retains the archive's SHA-512 and SHA-256 digests. npm downstream locks record the archive integrity; Bun 1.3.6 retains the exact URL without a digest. The asset CLI therefore checks fixed per-file hashes before every copy. This identifies the installed source and prebuilt artifacts; it is not an independent reproduction of the compiler environment or WebAssembly build. `node scripts/verify-sqlite-provenance.mjs` checks the archive integrity and compares every upstream file with the immutable source revision; `node scripts/copy-sqlite-assets.mjs --verify` verifies the installed dependency without network access. + +Keep `LICENSE`, the full source `ACKNOWLEDGMENTS.md`, and `PROVENANCE.md`. Do not substitute npm `wa-sqlite@1.0.0`: it is an unofficial publication with different contents and VFS API. + +## Verification + +Use Node 24 for disk tests (`node:sqlite`) and the repository's pinned Bun version to install dependencies: + +```sh +bun install --frozen-lockfile +bun run build:packages +cd packages/replicate +bun run test +bun run test:assets +bunx playwright install chromium +bun run test:browser +node scripts/test-native-bundle.cjs +``` + +The normal browser command checks candidate restart and offline behavior. To also run the +old-runtime write -> candidate reopen/write -> old-runtime reopen compatibility phases, provide +the original pinned source tree. The harness verifies every required legacy file against the +committed hash manifest before it builds the offline cache. For example, in PowerShell: + +```powershell +$compatRoot = Join-Path ([System.IO.Path]::GetTempPath()) ('replicate-sqlite-' + [guid]::NewGuid()) +$archive = Join-Path $compatRoot 'wa-sqlite.tar.gz' +New-Item -ItemType Directory -Path $compatRoot | Out-Null +Invoke-WebRequest ` + -Uri 'https://codeload.github.com/rhashimoto/wa-sqlite/tar.gz/55bf0b6646ac1bb81b00b9c25636b59a87e4a918' ` + -OutFile $archive +tar -xf $archive -C $compatRoot +$legacySource = Get-ChildItem -LiteralPath $compatRoot -Directory | + Where-Object Name -Like 'wa-sqlite-*' | + Select-Object -First 1 -ExpandProperty FullName +$env:REPLICATE_BASELINE_SQLITE_ASSETS = $legacySource +bun run --cwd packages/replicate test:browser +``` + +The compatibility run must report 12 check groups. It creates the database with the legacy +runtime, reopens and writes with the candidate, then reopens with the legacy runtime and verifies +both rows, cursor 2, no network requests, and no page errors. Remove the temporary directory and +environment variable after the run. + +Client tests use real SQLite files and actual collections/adapters with controlled transport. Component/server tests use convex-test, including seeded histories checked against independent source documents; they are not cloud deployment tests. The browser harness bundles package exports, precaches the actual SQLite/WASM graph, explicitly polls durable checkpoints, shuts down Chromium both with and without application cleanup, reopens the same profile offline, and verifies rows, removals, pending updates, and reset with no HTTP requests during offline phases. + +The native harness bundles the **full built client entry** through Expo Metro for Android, not only the SQLite adapter. With the locked lib0 0.2.117, the stock React Native conditional export requires unavailable `isomorphic-webcrypto`. Use a targeted Metro resolver mapping `lib0/webcrypto` to lib0's `webcrypto.js`, as in the repository Expo example; retain the application's native secure `crypto.getRandomValues`/`crypto.randomUUID` setup before importing Replicate. The mapping supplies no randomness and no WebCrypto implementation. Encrypted APIs additionally require a compatible secure WebCrypto implementation. No insecure fallback or obsolete isomorphic-webcrypto dependency is added. This verifies bundling, **not native-device execution**. diff --git a/docs/tests.md b/docs/tests.md index 8123210..6af3dc3 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -14,12 +14,12 @@ Replicate requires testing across multiple layers: ## Test Categories & Environments -| Category | Environment | Purpose | Tools | -| ---------------- | ------------ | --------------------------------- | ----------------------- | -| **Unit** | jsdom | Actor logic, validators, Yjs ops | Vitest + @effect/vitest | -| **Integration** | jsdom | Collection sync, offline→online | fake-indexeddb + Yjs | -| **E2E (Convex)** | edge-runtime | Component mutations, compaction | convex-test | -| **Browser** | Playwright | SQLite persistence, OPFS, Workers | @vitest/browser | +| Category | Environment | Purpose | Tools | +| ---------------- | ------------ | -------------------------------------- | ----------------------- | +| **Unit** | jsdom | Actor logic, validators, Yjs ops | Vitest + @effect/vitest | +| **Integration** | jsdom | Collection sync, offline→online | fake-indexeddb + Yjs | +| **E2E (Convex)** | edge-runtime | Component mutations, compaction | convex-test | +| **Browser** | Playwright | SQLite persistence, IndexedDB, Workers | @vitest/browser | ## Dependencies @@ -50,7 +50,7 @@ src/test/ │ ├── compaction.test.ts # scheduleCompaction + runCompaction │ └── presence.test.ts # Sessions, mark, presence └── browser/ # @vitest/browser (Playwright) - ├── sqlite.test.ts # wa-sqlite + OPFS + ├── sqlite.test.ts # wa-sqlite + IDBBatchAtomicVFS └── worker.test.ts # Web Worker communication ``` @@ -277,32 +277,42 @@ test('extracts prose text', () => { ```typescript // @vitest-environment browser -import { persistence } from '$/client/persistence'; +import { persistence } from '@trestleinc/replicate/client'; +import SqliteWorker from '@trestleinc/replicate/worker?worker'; describe('SQLite Persistence', () => { - let db: Awaited>; + let db: Awaited>; + let name: string; beforeEach(async () => { - db = await persistence.web.sqlite({ - name: `test-${crypto.randomUUID()}`, + name = `test-${crypto.randomUUID()}`; + db = await persistence.web.sqlite.create({ + name, + assetBase: '/replicate-sqlite', + worker: () => new SqliteWorker(), }); }); afterEach(async () => { - await db.close(); + await db.close!(); }); test('persists and retrieves data', async () => { - await db.kv.set('key', new Uint8Array([1, 2, 3])); + await db.kv.set('key', 'value'); const result = await db.kv.get('key'); - expect(result).toEqual(new Uint8Array([1, 2, 3])); + expect(result).toBe('value'); }); - test('uses OPFS via wa-sqlite', async () => { - // OPFS is automatically used when available - const estimate = await navigator.storage.estimate(); - expect(estimate.quota).toBeGreaterThan(0); + test('reopens IDBBatchAtomicVFS data from IndexedDB', async () => { + await db.kv.set('key', 'durable'); + await db.close!(); + db = await persistence.web.sqlite.create({ + name, + assetBase: '/replicate-sqlite', + worker: () => new SqliteWorker(), + }); + expect(await db.kv.get('key')).toBe('durable'); }); }); ``` @@ -408,7 +418,7 @@ export const createDocWithFields = (fields: Record): Y.Doc => { - Client → server → other clients - Offline accumulation → online sync 5. **Browser persistence** - Real browser APIs - - wa-sqlite + OPFS + - wa-sqlite + IDBBatchAtomicVFS (IndexedDB) - Web Worker communication ## Running Tests @@ -636,7 +646,7 @@ Rather than writing 100+ trivial tests, aim for: | **Compaction flow** | 3-5 tests | Scheduling, retry, peer safety, retain logic | | **Effect actors** | 4-6 tests | Batching, debounce, retry, shutdown cleanup | | **Integration sync** | 3-5 tests | Full client→server→client cycle, offline→online | -| **Browser persistence** | 2-4 tests | SQLite write/read, OPFS access | +| **Browser persistence** | 2-4 tests | SQLite write/read, IndexedDB reopen | **Total: ~17-28 deep tests** (not 50+ shallow ones) diff --git a/eslint.config.js b/eslint.config.js index 0b34019..2fa04b5 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -8,6 +8,7 @@ export default [ { ignores: [ '**/node_modules/**', + 'packages/replicate/assets/**', '**/.svelte-kit/**', '**/dist/**', '**/build/**', @@ -42,7 +43,7 @@ export default [ }, { // Allow CommonJS require() in config files - files: ['**/*.config.js', '**/*.config.cjs'], + files: ['**/*.config.js', '**/*.config.cjs', '**/scripts/*.cjs'], rules: { '@typescript-eslint/no-require-imports': 'off', }, diff --git a/package.json b/package.json index 37476cc..14a4df4 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,6 @@ "@tiptap/extension-placeholder": "^3.15.3", "lucide-react": "^0.562.0", "@lucide/svelte": "^0.562.0", - "wa-sqlite": "^1.0.0", "@electric-sql/pglite": "^0.3.14", "eslint": "^9.39.2", "@eslint/js": "^9.39.2", diff --git a/packages/replicate/assets/wa-sqlite/PROVENANCE.md b/packages/replicate/assets/wa-sqlite/PROVENANCE.md new file mode 100644 index 0000000..87220b4 --- /dev/null +++ b/packages/replicate/assets/wa-sqlite/PROVENANCE.md @@ -0,0 +1,18 @@ +# Runtime provenance + +The browser runtime is copied from the original wa-sqlite repository at Git +commit `07ad48cf2f682d279f9cd69818cc1d39b2ccda86`. Replicate installs that commit's +immutable GitHub codeload archive directly. The package metadata reports version +1.1.2, but this dependency is a commit-pinned source archive, not an npm release +or a claim about the repository's similarly named tags. + +The runtime files, MIT `LICENSE`, and complete `ACKNOWLEDGMENTS.md` listed in +`manifest.json` are byte-identical to that commit. The JavaScript loader was last +changed at `c17fe55df5d3b9da82ab34aa6b356a8a90eaa023`; the WebAssembly artifact was +last changed at `b0d5ff44548ce8a5c76f360a416c2d4321ed3c10`. The shipped module reports +SQLite 3.53.0, matching the Makefile. The installed archive has no runtime +dependencies or install, prepare, prepack, or build hooks. + +The archive integrity and per-file hashes identify the installed source and +prebuilt artifacts; they do not independently reproduce the compiler toolchain +or the WebAssembly build. diff --git a/packages/replicate/assets/wa-sqlite/manifest.json b/packages/replicate/assets/wa-sqlite/manifest.json new file mode 100644 index 0000000..0466325 --- /dev/null +++ b/packages/replicate/assets/wa-sqlite/manifest.json @@ -0,0 +1,42 @@ +{ + "package": "wa-sqlite", + "version": "1.1.2", + "archive": "https://codeload.github.com/rhashimoto/wa-sqlite/tar.gz/07ad48cf2f682d279f9cd69818cc1d39b2ccda86", + "archiveIntegrity": "sha512-CUMpsw8ldbamBHtCg9yPLsh1U5/jaCA4EkkZdABhSdaVq0Bie2lgVzYaMzQbqNMx8m/qWfvdoJ7rqFY7dK5aAw==", + "archiveSha256": "1579819cd2bbc4cd6cf71f591b8a1a45fb9a208e4e279d09c7226575c7e03811", + "repository": "https://github.com/rhashimoto/wa-sqlite", + "revision": "07ad48cf2f682d279f9cd69818cc1d39b2ccda86", + "artifactRevisions": { + "dist/wa-sqlite-async.mjs": "c17fe55df5d3b9da82ab34aa6b356a8a90eaa023", + "dist/wa-sqlite-async.wasm": "b0d5ff44548ce8a5c76f360a416c2d4321ed3c10" + }, + "sqliteVersion": "3.53.0", + "dependencyFiles": [ + "dist/wa-sqlite-async.mjs", + "dist/wa-sqlite-async.wasm", + "src/examples/IDBBatchAtomicVFS.js", + "src/FacadeVFS.js", + "src/VFS.js", + "src/WebLocksMixin.js", + "src/sqlite-api.js", + "src/sqlite-constants.js", + "LICENSE", + "ACKNOWLEDGMENTS.md" + ], + "packageFiles": [ + "PROVENANCE.md" + ], + "sha256": { + "dist/wa-sqlite-async.mjs": "4ac8be5305557ac06f08b70beae24b4e55ba21993b608c49b3531db1cf981a01", + "dist/wa-sqlite-async.wasm": "6bfcf02fe6c30eb05784850d985f37058475fce686cdcdc4322c8192c5a43722", + "src/examples/IDBBatchAtomicVFS.js": "809018e8976a8001bfac6062b1faa2a75333bd004885c8918f926c9e596e9cc6", + "src/FacadeVFS.js": "6de41eee28f7d4279d409e275b3e25fd5e7837163386e4e8f2c379cd9f72a60c", + "src/VFS.js": "55e7455e2ec0000cc4ba7a8bf7882f414a40f2a0ba6eb4c4ac847f379f54d92f", + "src/WebLocksMixin.js": "0ee2f39f43b3a8d70e94c0e5a2bac97f2c55416830183d4810caf28b2cec7375", + "src/sqlite-api.js": "6a18bc86fa5a2ef300c7a7a84bfc762caff396ec56af17fcc4885d20189fe54c", + "src/sqlite-constants.js": "f7b570f0c39e4b54c99f2598b6ad42018d4d8bf791861ede20d295628ae49f8f", + "LICENSE": "547f9e97e461bd369cc64b6fc12d724e8b02764b8db7812691f29f0ec572be79", + "ACKNOWLEDGMENTS.md": "5dae12fa0c82fe1a6a70fcf6b765298b5f32b7d32449f4d72ae1c12468457762", + "PROVENANCE.md": "e84891706b92843050803ffd44da64ddac9a9d5f875a69daea6aa7eb61a2789d" + } +} diff --git a/packages/replicate/package.json b/packages/replicate/package.json index 95e8590..2e8a6ff 100644 --- a/packages/replicate/package.json +++ b/packages/replicate/package.json @@ -21,7 +21,9 @@ "dist", "LICENSE", "README.md", - "src" + "src", + "assets", + "scripts/copy-sqlite-assets.mjs" ], "type": "module", "main": "./dist/shared/index.js", @@ -72,11 +74,15 @@ "lint": "eslint .", "lint:fix": "eslint . --fix", "format": "prettier --write .", - "format:check": "prettier --check ." + "format:check": "prettier --check .", + "test:browser": "node scripts/test-offline-browser.mjs", + "test:assets": "node scripts/copy-sqlite-assets.mjs --verify", + "test:packed-consumer": "node scripts/test-packed-consumer.mjs" }, "dependencies": { "@logtape/logtape": "catalog:", - "@logtape/pretty": "catalog:" + "@logtape/pretty": "catalog:", + "wa-sqlite": "https://codeload.github.com/rhashimoto/wa-sqlite/tar.gz/07ad48cf2f682d279f9cd69818cc1d39b2ccda86" }, "devDependencies": { "@convex-dev/eslint-plugin": "catalog:", @@ -104,23 +110,23 @@ "tsdown": "0.20.0-beta.3", "typescript": "catalog:", "typescript-eslint": "catalog:", - "vitest": "catalog:" + "vitest": "catalog:", + "vite": "catalog:" }, "peerDependencies": { "@op-engineering/op-sqlite": "15.1.14", "@tanstack/db": "^0.5.20", "convex": "^1.31.4", "lib0": "^0.2.0", - "wa-sqlite": "1.0.0", "y-protocols": "1.0.7", "yjs": "13.6.29" }, "peerDependenciesMeta": { - "wa-sqlite": { - "optional": true - }, "@op-engineering/op-sqlite": { "optional": true } + }, + "bin": { + "replicate-sqlite-assets": "./scripts/copy-sqlite-assets.mjs" } } diff --git a/packages/replicate/scripts/copy-sqlite-assets.mjs b/packages/replicate/scripts/copy-sqlite-assets.mjs new file mode 100755 index 0000000..6fde328 --- /dev/null +++ b/packages/replicate/scripts/copy-sqlite-assets.mjs @@ -0,0 +1,49 @@ +#!/usr/bin/env node +import { copyFile, mkdir, readFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; + +const packageSource = fileURLToPath(new URL('../assets/wa-sqlite/', import.meta.url)); +const manifestPath = resolve(packageSource, 'manifest.json'); +const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); +const require = createRequire(import.meta.url); +const dependencyManifestPath = require.resolve(`${manifest.package}/package.json`); +const dependencySource = dirname(dependencyManifestPath); +const dependencyManifest = JSON.parse(await readFile(dependencyManifestPath, 'utf8')); +if (dependencyManifest.name !== manifest.package || dependencyManifest.version !== manifest.version) { + throw new Error( + `Expected ${manifest.package}@${manifest.version}, found ${dependencyManifest.name}@${dependencyManifest.version}` + ); +} + +const dependencyFiles = new Set(manifest.dependencyFiles); +const packageFiles = new Set(manifest.packageFiles); +for (const [file, expected] of Object.entries(manifest.sha256)) { + const source = dependencyFiles.has(file) + ? resolve(dependencySource, file) + : packageFiles.has(file) + ? resolve(packageSource, file) + : undefined; + if (!source) throw new Error(`SQLite asset has no declared source: ${file}`); + const actual = createHash('sha256') + .update(await readFile(source)) + .digest('hex'); + if (actual !== expected) throw new Error(`SQLite asset checksum mismatch: ${file}`); +} +if (process.argv[2] !== '--verify') { + if (!process.argv[2]) + throw new Error('Usage: replicate-sqlite-assets | --verify'); + const destination = resolve(process.argv[2]); + for (const file of Object.keys(manifest.sha256)) { + const source = dependencyFiles.has(file) + ? resolve(dependencySource, file) + : resolve(packageSource, file); + const target = resolve(destination, file); + await mkdir(dirname(target), { recursive: true }); + await copyFile(source, target); + } + await mkdir(destination, { recursive: true }); + await copyFile(manifestPath, resolve(destination, 'manifest.json')); +} diff --git a/packages/replicate/scripts/test-native-bundle.cjs b/packages/replicate/scripts/test-native-bundle.cjs new file mode 100644 index 0000000..ea38390 --- /dev/null +++ b/packages/replicate/scripts/test-native-bundle.cjs @@ -0,0 +1,57 @@ +const { createRequire } = require('node:module'); +const { mkdtemp, rm, readFile } = require('node:fs/promises'); +const { tmpdir } = require('node:os'); +const path = require('node:path'); +const assert = require('node:assert/strict'); + +(async () => { + const packageRoot = path.resolve(__dirname, '..'); + const repo = path.resolve(packageRoot, '../..'); + const expoRequire = createRequire(path.join(repo, 'apps/expo/package.json')); + const { getDefaultConfig } = expoRequire('expo/metro-config'); + const metro = createRequire(expoRequire.resolve('expo/package.json'))('metro'); + const fixture = path.join(packageRoot, 'test/native'); + const config = getDefaultConfig(path.join(repo, 'apps/expo')); + config.projectRoot = fixture; + config.watchFolders = [repo]; + config.resolver.nodeModulesPaths = [ + path.join(packageRoot, 'node_modules'), + path.join(repo, 'apps/expo/node_modules'), + path.join(repo, 'node_modules'), + path.join(repo, 'node_modules/.bun/node_modules'), + ]; + config.resolver.extraNodeModules = { '@trestleinc/replicate': packageRoot }; + const lib0Root = path.dirname(require.resolve('lib0/package.json', { paths: [packageRoot] })); + // lib0's RN conditional export requires obsolete isomorphic-webcrypto. + // Use its browser entry with the application's native CSPRNG installed first. + config.resolver.resolveRequest = (context, name, platform) => { + if (name === 'lib0/webcrypto') + return { type: 'sourceFile', filePath: path.join(lib0Root, 'webcrypto.js') }; + return context.resolveRequest(context, name, platform); + }; + config.maxWorkers = 2; + const directory = await mkdtemp(path.join(tmpdir(), 'replicate-native-')); + try { + const output = path.join(directory, 'android.js'); + await metro.runBuild(config, { + entry: path.join(fixture, 'index.js'), + platform: 'android', + dev: false, + minify: false, + out: output, + }); + const bundle = await readFile(output, 'utf8'); + assert.ok(bundle.includes('replicateNativeGraph')); + assert.ok(bundle.includes('PersistenceProvider.storeUpdate')); + assert.ok(!bundle.includes('wa-sqlite-async.mjs')); + assert.ok(!bundle.includes('IDBBatchAtomicVFS.create')); + process.stdout.write( + 'Full client graph bundled for Android without browser SQLite runtime (no device execution)\n' + ); + } finally { + await rm(directory, { recursive: true, force: true }); + } +})().catch((error) => { + process.stderr.write(String(error) + '\n'); + process.exitCode = 1; +}); diff --git a/packages/replicate/scripts/test-offline-browser.mjs b/packages/replicate/scripts/test-offline-browser.mjs new file mode 100644 index 0000000..e5889d6 --- /dev/null +++ b/packages/replicate/scripts/test-offline-browser.mjs @@ -0,0 +1,228 @@ +import { build } from 'vite'; +import { chromium } from 'playwright'; +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { createServer } from 'node:http'; +import { copyFile, mkdir, mkdtemp, readdir, readFile, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve, extname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; +import assert from 'node:assert/strict'; +import * as Y from 'yjs'; + +const root = fileURLToPath(new URL('../', import.meta.url)); +const temp = await mkdtemp(join(tmpdir(), 'replicate-browser-')); +const site = join(temp, 'site'); +const profile = join(temp, 'profile'); +let context; +let server; +let checks = 0; +async function waitForCursor(page, expected) { + const deadline = Date.now() + 30000; + while ((await page.evaluate(() => window.replica.cursor())) !== expected) { + if (Date.now() > deadline) throw new Error(`Cursor did not reach ${expected}`); + await new Promise((resolve) => setTimeout(resolve, 20)); + } +} +try { + await build({ + configFile: false, + root: resolve(root, 'test/browser'), + logLevel: 'warn', + build: { outDir: site, emptyOutDir: true }, + }); + const copyAssets = resolve(root, 'scripts/copy-sqlite-assets.mjs'); + await promisify(execFile)(process.execPath, [copyAssets, join(site, 'sqlite')]); + const baselineAssets = process.env.REPLICATE_BASELINE_SQLITE_ASSETS; + if (baselineAssets) { + const baselineManifest = JSON.parse( + await readFile(resolve(root, 'test/browser/baseline-wa-sqlite-manifest.json'), 'utf8') + ); + for (const [file, expected] of Object.entries(baselineManifest.sha256)) { + const source = resolve(baselineAssets, file); + const bytes = await readFile(source); + assert.equal(createHash('sha256').update(bytes).digest('hex'), expected, file); + const target = join(site, 'sqlite-baseline', file); + await mkdir(dirname(target), { recursive: true }); + await copyFile(source, target); + } + await writeFile( + join(site, 'sqlite-baseline', 'manifest.json'), + JSON.stringify(baselineManifest, null, 2) + ); + await promisify(execFile)(process.execPath, [copyAssets, join(site, 'sqlite-candidate')]); + } + const paths = (await readdir(site, { recursive: true, withFileTypes: true })) + .filter((entry) => entry.isFile()) + .map( + (entry) => + '/' + + join(entry.parentPath, entry.name) + .slice(site.length + 1) + .replaceAll('\\', '/') + ); + await writeFile( + join(site, 'sw.js'), + `self.addEventListener('install', event => event.waitUntil(caches.open('replica-v1').then(cache => cache.addAll(${JSON.stringify(['/', ...paths])})).then(() => self.skipWaiting()))); self.addEventListener('activate', event => event.waitUntil(self.clients.claim())); self.addEventListener('fetch', event => event.respondWith(caches.match(event.request).then(hit => hit || fetch(event.request))));` + ); + let requests = 0; + server = createServer(async (req, res) => { + requests++; + try { + const name = req.url === '/' ? '/index.html' : new URL(req.url, 'http://local').pathname; + const path = resolve(site, '.' + name); + if (!path.startsWith(site)) throw new Error('Invalid path'); + const data = await readFile(path); + res.setHeader( + 'Content-Type', + { + '.wasm': 'application/wasm', + '.js': 'application/javascript', + '.mjs': 'application/javascript', + '.html': 'text/html', + }[extname(path)] ?? 'application/octet-stream' + ); + res.end(data); + } catch { + res.writeHead(404); + res.end(); + } + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const url = `http://127.0.0.1:${server.address().port}/`; + const errors = []; + async function launch(offline = false, fragment = '') { + context = await chromium.launchPersistentContext(profile, { headless: true, offline }); + const page = await context.newPage(); + page.on('pageerror', (error) => errors.push(error.message)); + await page.goto(url + fragment); + await page.waitForFunction(() => !!window.replica); + await page.waitForFunction(() => !!navigator.serviceWorker.controller); + return page; + } + async function restart(page, { closeApplication = false, fragment = '' } = {}) { + if (closeApplication) await page.evaluate(() => window.replica.close()); + await context.close(); + return launch(true, fragment); + } + let page = await launch(); + assert.equal(await page.evaluate(() => window.replica.calls()), 0); + checks++; + await page.evaluate(async () => { + await window.replica.connect(); + window.replica.emit('a', 1); + }); + await waitForCursor(page, 1); + checks++; + page = await restart(page, { closeApplication: true }); + assert.equal(await page.evaluate(() => window.replica.get('a').title), 'persisted in browser'); + checks++; + assert.equal(await page.evaluate(() => window.replica.calls()), 0); + checks++; + const requestCount = requests; + await page.evaluate(async () => { + await window.replica.connect(); + window.replica.emit('a', 2, false); + }); + await page.waitForFunction(() => !window.replica.get('a')); + page = await restart(page); + assert.equal(await page.evaluate(() => window.replica.get('a')), undefined); + checks++; + await page.evaluate(async () => { + await window.replica.connect(); + window.replica.emit('b', 3); + }); + await page.waitForFunction(() => !!window.replica.get('b')); + page = await restart(page); + await page.evaluate(async () => { + window.replica.absent(); + await window.replica.connect(); + }); + await page.waitForFunction(() => !window.replica.get('b')); + page = await restart(page); + assert.equal(await page.evaluate(() => window.replica.get('b')), undefined); + checks++; + const doc = new Y.Doc(); + doc.getMap('fields').set('title', 'base'); + const base = Y.encodeStateAsUpdateV2(doc); + const vector = Y.encodeStateVector(doc); + doc.getMap('fields').set('title', 'after restart'); + const delta = Y.encodeStateAsUpdateV2(doc, vector); + doc.destroy(); + await page.evaluate( + async (raw) => { + await window.replica.connect(); + window.replica.emit('pending', 4, true, raw); + }, + [...delta] + ); + await waitForCursor(page, 4); + page = await restart(page); + await page.evaluate( + async (raw) => { + await window.replica.connect(); + window.replica.emit('pending', 5, true, raw); + }, + [...base] + ); + await page.waitForFunction(() => window.replica.get('pending')?.title === 'after restart'); + checks++; + await page.evaluate(() => window.replica.reset()); + page = await restart(page); + assert.equal(await page.evaluate(() => window.replica.get('pending')), undefined); + checks++; + assert.equal(requests, requestCount); + assert.deepEqual(errors, []); + checks++; + if (baselineAssets) { + await page.evaluate(() => window.replica.close()); + await context.close(); + const compatibilityRequests = requests; + const database = 'runtime-compatibility'; + const oldFragment = `#assetBase=/sqlite-baseline&database=${database}`; + const candidateFragment = `#assetBase=/sqlite-candidate&database=${database}`; + page = await launch(true, oldFragment); + await page.evaluate(async () => { + await window.replica.connect(); + window.replica.emit('old-write', 1); + }); + await waitForCursor(page, 1); + page = await restart(page, { closeApplication: true, fragment: candidateFragment }); + assert.equal( + await page.evaluate(() => window.replica.get('old-write')?.title), + 'persisted in browser' + ); + checks++; + await page.evaluate(async () => { + await window.replica.connect(); + window.replica.emit('candidate-write', 2); + }); + await waitForCursor(page, 2); + page = await restart(page, { closeApplication: true, fragment: oldFragment }); + assert.equal( + await page.evaluate(() => window.replica.get('old-write')?.title), + 'persisted in browser' + ); + assert.equal( + await page.evaluate(() => window.replica.get('candidate-write')?.title), + 'persisted in browser' + ); + assert.equal(await page.evaluate(() => window.replica.cursor()), 2); + assert.equal(requests, compatibilityRequests); + assert.deepEqual(errors, []); + checks++; + } + assert.deepEqual(await page.evaluate(() => window.replica.closeRace()), { + acceptedCompleted: true, + closeMessages: 1, + lateRejected: true, + }); + checks++; + assert.deepEqual(errors, []); + process.stdout.write(`${checks} real Chromium offline/process-restart checks passed\n`); +} finally { + await context?.close(); + await new Promise((resolve) => (server ? server.close(resolve) : resolve())); + await rm(temp, { recursive: true, force: true }); +} diff --git a/packages/replicate/scripts/test-packed-consumer.mjs b/packages/replicate/scripts/test-packed-consumer.mjs new file mode 100644 index 0000000..bce582d --- /dev/null +++ b/packages/replicate/scripts/test-packed-consumer.mjs @@ -0,0 +1,151 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { access, mkdtemp, mkdir, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; + +const tarballArgument = process.env.REPLICATE_PACKAGE_TARBALL; +if (!tarballArgument) { + throw new Error('Set REPLICATE_PACKAGE_TARBALL to a packed @trestleinc/replicate tarball'); +} +const tarball = isAbsolute(tarballArgument) + ? tarballArgument + : resolve(process.cwd(), tarballArgument); +const npmCandidates = [ + process.env.REPLICATE_NPM_CLI, + process.env.npm_execpath?.endsWith('npm-cli.js') ? process.env.npm_execpath : undefined, + join(dirname(process.execPath), 'node_modules', 'npm', 'bin', 'npm-cli.js'), + join(dirname(dirname(process.execPath)), 'lib', 'node_modules', 'npm', 'bin', 'npm-cli.js'), + '/usr/local/lib/node_modules/npm/bin/npm-cli.js', + '/usr/lib/node_modules/npm/bin/npm-cli.js', + '/usr/share/nodejs/npm/bin/npm-cli.js', +].filter(Boolean); +let npmCli; +for (const candidate of npmCandidates) { + try { + await access(candidate); + npmCli = candidate; + break; + } catch { + // Try the next known npm layout. + } +} +if (!npmCli) throw new Error('Could not locate npm-cli.js; set REPLICATE_NPM_CLI'); +const npm = (args, options) => promisify(execFile)(process.execPath, [npmCli, ...args], options); + +let cleanupDirectory; +try { + cleanupDirectory = await mkdtemp(join(tmpdir(), 'replicate packed consumer ')); + const directory = await realpath(cleanupDirectory); + const consumer = join(directory, 'consumer with spaces'); + const destination = join(consumer, 'public assets', 'sqlite'); + await mkdir(consumer, { recursive: true }); + await writeFile( + join(consumer, 'package.json'), + JSON.stringify({ name: 'replicate-packed-consumer', private: true }, null, 2) + ); + await npm( + [ + 'install', + '--ignore-scripts', + '--omit=optional', + '--no-audit', + '--no-fund', + tarball, + 'typescript@5.9.3', + ], + { cwd: consumer } + ); + + const packageRoot = join(consumer, 'node_modules', '@trestleinc', 'replicate'); + const packageRequire = createRequire(join(packageRoot, 'package.json')); + const dependencyManifestPath = packageRequire.resolve('wa-sqlite/package.json'); + assert.ok(dependencyManifestPath.startsWith(directory)); + assert.equal(JSON.parse(await readFile(dependencyManifestPath, 'utf8')).version, '1.1.2'); + const specifiers = [ + '@trestleinc/replicate', + '@trestleinc/replicate/client', + '@trestleinc/replicate/server', + '@trestleinc/replicate/convex.config', + '@trestleinc/replicate/worker', + ]; + const resolver = join(consumer, 'resolve-exports.mjs'); + await writeFile( + resolver, + `const specifiers = ${JSON.stringify(specifiers)}; +const resolved = specifiers.map((specifier) => import.meta.resolve(specifier)); +for (const specifier of specifiers.slice(0, -1)) await import(specifier); +process.stdout.write(JSON.stringify(resolved)); +` + ); + const { stdout: resolvedOutput } = await promisify(execFile)(process.execPath, [resolver], { + cwd: consumer, + }); + for (const url of JSON.parse(resolvedOutput)) { + assert.ok(fileURLToPath(url).startsWith(packageRoot), url); + } + + await writeFile( + join(consumer, 'consumer-exports.ts'), + `import type * as shared from '@trestleinc/replicate'; +import type * as client from '@trestleinc/replicate/client'; +import type * as server from '@trestleinc/replicate/server'; +import type * as config from '@trestleinc/replicate/convex.config'; +type ResolvedExports = [typeof shared, typeof client, typeof server, typeof config]; +declare const resolvedExports: ResolvedExports; +void resolvedExports; +` + ); + await writeFile( + join(consumer, 'tsconfig.json'), + JSON.stringify( + { + compilerOptions: { + module: 'NodeNext', + moduleResolution: 'NodeNext', + noUncheckedSideEffectImports: true, + noEmit: true, + skipLibCheck: false, + strict: true, + target: 'ES2022', + }, + files: ['consumer-exports.ts'], + }, + null, + 2 + ) + ); + const consumerRequire = createRequire(join(consumer, 'package.json')); + await promisify(execFile)(process.execPath, [consumerRequire.resolve('typescript/bin/tsc')], { + cwd: consumer, + }); + + const run = () => + npm(['exec', '--', 'replicate-sqlite-assets', destination], { + cwd: consumer, + }); + await run(); + const manifest = JSON.parse(await readFile(join(destination, 'manifest.json'), 'utf8')); + for (const [file, expected] of Object.entries(manifest.sha256)) { + const bytes = await readFile(join(destination, file)); + assert.equal(createHash('sha256').update(bytes).digest('hex'), expected, file); + } + + await writeFile(join(destination, 'dist', 'wa-sqlite-async.wasm'), 'stale runtime'); + await writeFile(join(destination, 'consumer-owned.txt'), 'preserved'); + await run(); + assert.equal(await readFile(join(destination, 'consumer-owned.txt'), 'utf8'), 'preserved'); + assert.equal( + createHash('sha256') + .update(await readFile(join(destination, 'dist', 'wa-sqlite-async.wasm'))) + .digest('hex'), + manifest.sha256['dist/wa-sqlite-async.wasm'] + ); + process.stdout.write('Packed consumer installed dependency and copied verified SQLite assets\n'); +} finally { + if (cleanupDirectory) await rm(cleanupDirectory, { recursive: true, force: true }); +} diff --git a/packages/replicate/scripts/verify-sqlite-provenance.mjs b/packages/replicate/scripts/verify-sqlite-provenance.mjs new file mode 100644 index 0000000..7a9ad40 --- /dev/null +++ b/packages/replicate/scripts/verify-sqlite-provenance.mjs @@ -0,0 +1,29 @@ +// Read-only comparison with the pinned original source; does not regenerate compiler output. +import { readFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +const manifest = JSON.parse( + await readFile(new URL('../assets/wa-sqlite/manifest.json', import.meta.url), 'utf8') +); +const archiveResponse = await fetch(manifest.archive); +if (!archiveResponse.ok) throw new Error(`source archive: HTTP ${archiveResponse.status}`); +const archive = new Uint8Array(await archiveResponse.arrayBuffer()); +const archiveSha512 = `sha512-${createHash('sha512').update(archive).digest('base64')}`; +if (archiveSha512 !== manifest.archiveIntegrity) throw new Error('source archive integrity mismatch'); +if (createHash('sha256').update(archive).digest('hex') !== manifest.archiveSha256) { + throw new Error('source archive SHA-256 mismatch'); +} + +for (const file of manifest.dependencyFiles) { + const response = await fetch( + `${manifest.repository.replace('github.com', 'raw.githubusercontent.com')}/${manifest.revision}/${file}` + ); + if (!response.ok) throw new Error(`${file}: HTTP ${response.status}`); + const hash = manifest.sha256[file]; + if ( + createHash('sha256') + .update(new Uint8Array(await response.arrayBuffer())) + .digest('hex') !== hash + ) { + throw new Error(`Upstream checksum mismatch: ${file}`); + } +} diff --git a/packages/replicate/src/client/bindings/counter.ts b/packages/replicate/src/client/bindings/counter.ts index d7a484d..7e65a9f 100644 --- a/packages/replicate/src/client/bindings/counter.ts +++ b/packages/replicate/src/client/bindings/counter.ts @@ -66,7 +66,6 @@ export interface CounterBindingConfig { getMaterial: () => Record | null; } - /** * Compute counter value by summing all deltas. */ @@ -101,7 +100,17 @@ function createSyncFn( * Create a counter binding. */ export function createCounterBinding(config: CounterBindingConfig): CounterBinding { - const { collection, document, field, array, ydoc, collectionRef, clientId, debounceMs, getMaterial } = config; + const { + collection, + document, + field, + array, + ydoc, + collectionRef, + clientId, + debounceMs, + getMaterial, + } = config; if (!hasContext(collection)) { logger.warn('Cannot create counter binding - collection not initialized', { @@ -138,7 +147,7 @@ export function createCounterBinding(config: CounterBindingConfig): CounterBindi * Observer handler for Y.Array changes. */ const observerHandler = (_event: Y.YArrayEvent, transaction: Y.Transaction) => { - if (transaction.origin === SERVER_ORIGIN) { + if (transaction.origin === SERVER_ORIGIN || transaction.origin === 'replicate-persisted') { notifySubscribers(); return; } diff --git a/packages/replicate/src/client/bindings/set.ts b/packages/replicate/src/client/bindings/set.ts index 0318d72..22758ad 100644 --- a/packages/replicate/src/client/bindings/set.ts +++ b/packages/replicate/src/client/bindings/set.ts @@ -64,7 +64,6 @@ export interface SetBindingConfig { getMaterial: () => Record | null; } - /** * Create sync function for set. */ @@ -168,7 +167,7 @@ export function createSetBinding(config: SetBindingConfig): SetBinding transaction: Y.Transaction ) => { notifySubscribers(); - if (transaction.origin !== SERVER_ORIGIN) { + if (transaction.origin !== SERVER_ORIGIN && transaction.origin !== 'replicate-persisted') { sync.onLocalChange(); } }; diff --git a/packages/replicate/src/client/collection.ts b/packages/replicate/src/client/collection.ts index e5c66dc..1532ad6 100644 --- a/packages/replicate/src/client/collection.ts +++ b/packages/replicate/src/client/collection.ts @@ -1,3 +1,5 @@ +import { cleanupSyncManager } from './bindings/sync-registry'; +import { createConnection, type ConnectionOptions } from './services/connection'; import * as Y from 'yjs'; import type { Persistence, PersistenceProvider } from '$/client/persistence/types'; import type { ConvexClient } from 'convex/browser'; @@ -26,16 +28,12 @@ import { createDocumentManager, serializeDocument, extractAllDocuments, + isDocumentDeleted, } from '$/client/documents'; import { createDeleteDelta, applyDeleteMarkerToDoc } from '$/client/deltas'; import * as prose from '$/client/prose'; import { detectYjsType, isYjsAbstractType } from '$/client/merge'; import { getLogger } from '$/shared/logger'; -import { - createTransactionCoordinator, - type TransactionCoordinator, - type StagedChange, -} from '$/client/services/transaction'; import { createSyncQueue, type SyncQueue } from '$/client/services/sync-queue'; import { initContext, @@ -67,23 +65,6 @@ const logger = getLogger(['replicate', 'collection']); // Helper Functions // ============================================================================ -/** - * Higher-order function for error isolation in handlers. - * Catches errors and calls errorHandler, returning its result. - */ -const withErrorIsolation = ( - fn: (...args: T) => R, - errorHandler: (error: unknown, ...args: T) => R -) => { - return (...args: T): R => { - try { - return fn(...args); - } catch (error) { - return errorHandler(error, ...args); - } - }; -}; - import type { ProseFields } from '$/shared'; interface HttpError extends Error { @@ -164,10 +145,8 @@ interface ConvexCollectionApi { session: FunctionReference<'query'>; } -export interface ConvexCollectionConfig extends Omit< - BaseCollectionConfig, - 'schema' -> { +export interface ConvexCollectionConfig + extends Omit, 'schema'>, ConnectionOptions { validator?: GenericValidator; convexClient: ConvexClient; api: ConvexCollectionApi; @@ -274,6 +253,9 @@ export interface DocumentHandle { } interface ConvexCollectionExtensions { + connect(): Promise; + disconnect(): Promise; + reset(): Promise; doc(id: string): DocumentHandle; readonly session: SessionAPI; } @@ -289,13 +271,82 @@ export function convexCollectionOptions( validator, getKey, material, - convexClient, + convexClient: rawClient, api, persistence, user: userGetter, anonymousPresence, } = config; + const connection = createConnection(rawClient, config); + const convexClient = connection.client; + const ensureGeneration = (token: number) => { + if (stopped || token !== connection.generation) + throw new Error('Operation belongs to a disconnected generation'); + }; + const bindings = new Set<{ destroy(): void }>(); + const trackBinding = (binding: T): T => { + const token = connection.generation; + let destroyed = false; + const wrapped = new Proxy(binding, { + get(target, property) { + if (property === 'destroy') + return () => { + if (destroyed) return; + destroyed = true; + target.destroy(); + bindings.delete(wrapped); + }; + const value = Reflect.get(target, property); + if (typeof value !== 'function') return value; + return (...args: unknown[]) => { + ensureGeneration(token); + if (destroyed) throw new Error('Binding has been destroyed'); + return value.apply(target, args); + }; + }, + }); + bindings.add(wrapped); + return wrapped; + }; + const assertWritable = () => { + if (config.readOnly) throw new Error('Collection is read-only'); + }; + let startNetwork: (() => Promise) | undefined; + let work: Promise = Promise.resolve(); + let stopped = false; + let desiredConnection = config.networkMode !== 'manual'; + let disconnecting: Promise = Promise.resolve(); + const revisions = new Map(); + const durableVectors = new Map(); + const pendingCreates = new Set(); + const fullSyncRequired = new Map(); + let fullSyncEpoch = 0; + const pendingCreateKey = (id: string) => `pending-create:${collection}:${id}`; + const fullSyncKey = (id: string) => `full-sync-required:${collection}:${id}`; + let retryTimer: ReturnType | undefined; + let localReady: Promise = Promise.resolve(); + const serialize = (operation: () => Promise) => { + const next = work.then(operation); + work = next.catch(() => {}); + return next; + }; + const acknowledgeCreate = (id: string, token: number) => + serialize(async () => { + ensureGeneration(token); + await persistence.kv.del(pendingCreateKey(id)); + ensureGeneration(token); + pendingCreates.delete(id); + }); + const acknowledgeFullSync = (id: string, epoch: number, token: number, revision: number) => + serialize(async () => { + ensureGeneration(token); + if (fullSyncRequired.get(id) !== epoch || revisions.get(id) !== revision) return; + await persistence.kv.del(fullSyncKey(id)); + ensureGeneration(token); + if (fullSyncRequired.get(id) === epoch && revisions.get(id) === revision) + fullSyncRequired.delete(id); + }); const functionPath = getFunctionName(api.delta); const collection = functionPath.split(':')[0]; if (!collection) { @@ -322,6 +373,7 @@ export function convexCollectionOptions( field: ProseFields, options?: ProseOptions ): Promise { + assertWritable(); const fieldStr = field; if (!proseFieldSet.has(fieldStr)) { @@ -414,12 +466,18 @@ export function convexCollectionOptions( client: storedClientId, ydoc: subdoc, syncReady: ctx.synced, + durableVector: () => durableVectors.get(document), user: options?.user ?? ctx.userGetter, throttleMs: options?.throttleMs, anonymousPresence: ctx.anonymousPresence, }); } + if (presenceProvider) { + const providers = editorPresenceProviders.get(document) ?? new Set(); + providers.add(presenceProvider); + editorPresenceProviders.set(document, providers); + } const binding: EditorBinding = { fragment, provider: presenceProvider @@ -436,10 +494,11 @@ export function convexCollectionOptions( destroy() { presenceProvider?.destroy(); + if (presenceProvider) editorPresenceProviders.get(document)?.delete(presenceProvider); }, }; - return binding; + return trackBinding(binding); }, /** @@ -450,6 +509,7 @@ export function convexCollectionOptions( field: string, options?: { debounceMs?: number } ): Promise { + assertWritable(); const fieldStr = String(field); const crdtInfo = crdtFields.get(fieldStr); @@ -490,18 +550,20 @@ export function convexCollectionOptions( throw new Error(`Client ID not available`); } - return createCounterBinding({ - collection, - document, - field: fieldStr, - array, - ydoc: subdoc, - ymap: fieldsMap, - collectionRef, - clientId: ctx.clientId, - debounceMs: options?.debounceMs, - getMaterial: () => serializeDocument(ctx.docManager, document), - }); + return trackBinding( + createCounterBinding({ + collection, + document, + field: fieldStr, + array, + ydoc: subdoc, + ymap: fieldsMap, + collectionRef, + clientId: ctx.clientId, + debounceMs: options?.debounceMs, + getMaterial: () => serializeDocument(ctx.docManager, document), + }) + ); }, /** @@ -512,6 +574,7 @@ export function convexCollectionOptions( field: string, options?: { resolve?: (conflict: unknown) => T } ): Promise> { + assertWritable(); const fieldStr = String(field); const crdtInfo = crdtFields.get(fieldStr); @@ -549,11 +612,13 @@ export function convexCollectionOptions( (ctx.crdtResolvers?.get(fieldStr) as ((conflict: unknown) => T) | undefined) ?? ((c: { latest: () => T }) => c.latest()); - return createRegisterBinding({ - ymap: map, - clientId: ctx.clientId, - resolve, - }); + return trackBinding( + createRegisterBinding({ + ymap: map, + clientId: ctx.clientId, + resolve, + }) + ); }, /** @@ -568,6 +633,7 @@ export function convexCollectionOptions( debounceMs?: number; } ): Promise> { + assertWritable(); const fieldStr = String(field); const crdtInfo = crdtFields.get(fieldStr); @@ -604,25 +670,39 @@ export function convexCollectionOptions( throw new Error(`Client ID not available`); } - return createSetBinding({ - collection, - document, - field: fieldStr, - ymap: map, - ydoc: subdoc, - fieldsMap, - collectionRef, - clientId: ctx.clientId, - serialize: options.serialize, - deserialize: options.deserialize, - debounceMs: options.debounceMs, - getMaterial: () => serializeDocument(ctx.docManager, document), - }); + return trackBinding( + createSetBinding({ + collection, + document, + field: fieldStr, + ymap: map, + ydoc: subdoc, + fieldsMap, + collectionRef, + clientId: ctx.clientId, + serialize: options.serialize, + deserialize: options.deserialize, + debounceMs: options.debounceMs, + getMaterial: () => serializeDocument(ctx.docManager, document), + }) + ); }, }; const documentHandles = new Map>(); const presenceProviders = new Map(); + const editorPresenceProviders = new Map>(); + const removeDocument = async (id: string) => { + await docManager.remove(id); + await persistence.kv.del(fullSyncKey(id)); + fullSyncRequired.delete(id); + presenceProviders.get(id)?.destroy(); + presenceProviders.delete(id); + for (const provider of editorPresenceProviders.get(id) ?? []) provider.destroy(); + editorPresenceProviders.delete(id); + documentHandles.delete(id); + durableVectors.delete(id); + }; const getOrCreateDocumentHandle = (documentId: string): DocumentHandle => { let handle = documentHandles.get(documentId); @@ -649,6 +729,7 @@ export function convexCollectionOptions( client: ctx.clientId, ydoc: subdoc, syncReady: ctx.synced, + durableVector: () => durableVectors.get(documentId), user: ctx.userGetter, anonymousPresence: ctx.anonymousPresence, }); @@ -683,7 +764,7 @@ export function convexCollectionOptions( let sessionUnsubscribe: (() => void) | null = null; const initSessionSubscription = (): void => { - if (sessionUnsubscribe) return; + if (sessionUnsubscribe || !connection.active) return; const ctx = hasContext(collection) ? getContext(collection) : null; if (!ctx?.client || !ctx?.api?.session) return; @@ -721,6 +802,59 @@ export function convexCollectionOptions( }; const extensions: ConvexCollectionExtensions = { + async connect() { + const requestedGeneration = connection.generation; + desiredConnection = true; + await disconnecting; + await localReady; + if (stopped) throw new Error('Collection has been cleaned up'); + if (!desiredConnection || requestedGeneration !== connection.generation) return; + if (connection.active) return; + connection.connect(); + await startNetwork?.(); + }, + async disconnect() { + desiredConnection = false; + for (const binding of bindings) binding.destroy(); + prose.cleanup(collection); + cleanupSyncManager(collection); + clearTimeout(retryTimer); + connection.disconnect(); + sessionUnsubscribe?.(); + sessionUnsubscribe = null; + sessionCache = []; + for (const provider of presenceProviders.values()) provider.destroy(); + presenceProviders.clear(); + for (const providers of editorPresenceProviders.values()) + for (const provider of providers) provider.destroy(); + editorPresenceProviders.clear(); + documentHandles.clear(); + disconnecting = (async () => { + await localReady; + await work; + await docManager.flush(); + })(); + await disconnecting; + }, + async reset() { + const drained = extensions.disconnect(); + disconnecting = drained.then(() => + serialize(async () => { + for (const id of await persistence.listDocuments(collection)) docManager.getOrCreate(id); + for (const id of docManager.documents()) { + await removeDocument(id); + await persistence.kv.del(pendingCreateKey(id)); + } + for (const id of pendingCreates) await persistence.kv.del(pendingCreateKey(id)); + pendingCreates.clear(); + fullSyncRequired.clear(); + revisions.clear(); + await seqService.clear(collection); + ops.replace([]); + }) + ); + await disconnecting; + }, doc(id: string): DocumentHandle { return getOrCreateDocumentHandle(id); }, @@ -745,10 +879,19 @@ export function convexCollectionOptions( // Bound replicate operations - set during sync initialization // Used by onDelete and other handlers that need to sync with TanStack DB let ops: BoundReplicateOps = null as any; - - // Transaction coordinator for atomic client-side mutations with rollback - // Ensures delete operations are atomic: if server call fails, local state is rolled back - let txCoordinator: TransactionCoordinator | null = null; + const publishDocuments = (ids: Iterable) => { + const upserts: DataType[] = []; + const removals: DataType[] = []; + for (const id of ids) { + const item = isDocumentDeleted(docManager, id) + ? null + : serializeDocument(docManager, id, crdtFields); + if (item) upserts.push(item as DataType); + else removals.push({ id } as DataType); + } + if (removals.length) ops.delete(removals); + if (upserts.length) ops.upsert(upserts); + }; // Background sync queue for non-blocking server mutations const syncQueue: SyncQueue = createSyncQueue({ @@ -771,55 +914,61 @@ export function convexCollectionOptions( }); const recover = async (pushLocal = false): Promise => { + const token = connection.generation; const docIds = docManager.documents(); - if (docIds.length === 0) return; - - logger.debug('Starting recovery for documents', { collection, count: docIds.length }); - - const recoveryPromises = docIds.map(async (docId) => { - try { - const vector = docManager.encodeStateVector(docId); - const result = await convexClient.query(api.delta, { - document: docId, - vector: vector.buffer as ArrayBuffer, - }); - - if (result.mode === 'recovery' && result.diff) { - const update = new Uint8Array(result.diff); - docManager.applyUpdate(docId, update, YjsOrigin.Server); - logger.debug('Applied server diff during recovery', { document: docId, collection }); - } - - // Only push local state when explicitly requested (reconnection scenario) - // On init, we only pull server diff - pushing would flood the mutation queue - if (pushLocal) { - const ydoc = docManager.get(docId); - if (ydoc) { - const localState = Y.encodeStateAsUpdateV2(ydoc); + await Promise.all( + docIds.map(async (docId) => { + try { + const original = docManager.get(docId); + const revision = revisions.get(docId); + const vector = docManager.encodeStateVector(docId); + const result = await convexClient.query(api.delta, { + document: docId, + vector: vector.buffer as ArrayBuffer, + }); + let validRecovery = false; + await serialize(async () => { + if ( + !connection.active || + token !== connection.generation || + docManager.get(docId) !== original || + revisions.get(docId) !== revision + ) + return; + if (result.mode !== 'recovery') return; + validRecovery = true; + if (result.exists === false) { + if (!config.readOnly && (pendingCreates.has(docId) || result.known === false)) return; + await removeDocument(docId); + } else if (result.diff) { + await docManager.applyRemote(docId, new Uint8Array(result.diff)); + } else return; + publishDocuments([docId]); + }); + if ( + validRecovery && + revisions.get(docId) === revision && + (pushLocal || pendingCreates.has(docId)) && + !config.readOnly && + connection.active && + token === connection.generation + ) { + const ydoc = docManager.get(docId); const material = serializeDocument(docManager, docId); - - if (material && localState.length > 0) { + if (ydoc && material) await convexClient.mutation(api.replicate, { document: docId, - bytes: localState.buffer as ArrayBuffer, + bytes: Y.encodeStateAsUpdateV2(ydoc).buffer as ArrayBuffer, material, - type: 'update', + type: result.exists === false ? 'insert' : 'update', }); - logger.debug('Pushed local changes during recovery', { document: docId, collection }); - } + await acknowledgeCreate(docId, token); } + } catch (error) { + logger.warn('Recovery failed', { collection, document: docId, error }); } - } catch (error) { - logger.warn('Recovery failed for document', { - document: docId, - collection, - error: error instanceof Error ? error.message : String(error), - }); - } - }); - - await Promise.all(recoveryPromises); - logger.debug('Recovery completed', { collection, count: docIds.length }); + }) + ); }; const applyYjsInsert = (mutations: CollectionMutation[]): Uint8Array[] => { @@ -827,6 +976,7 @@ export function convexCollectionOptions( for (const mut of mutations) { const document = String(mut.key); + revisions.set(document, (revisions.get(document) ?? 0) + 1); const delta = docManager.transactWithDelta( document, (fieldsMap) => { @@ -868,6 +1018,7 @@ export function convexCollectionOptions( for (const mut of mutations) { const document = String(mut.key); + revisions.set(document, (revisions.get(document) ?? 0) + 1); const fieldsMap = docManager.getFields(document); if (!fieldsMap) { @@ -884,9 +1035,21 @@ export function convexCollectionOptions( (fields) => { const ctx = hasContext(collection) ? getContext(collection) : null; const effectiveClientId = ctx?.clientId ?? 'local'; + const modifiedKeys = new Set( + Object.entries(modifiedFields) + .filter(([key, value]) => crdtFields.has(key) || value !== undefined) + .map(([key]) => key) + ); + fields.forEach((_, key) => { + if (!crdtFields.has(key) && !modifiedKeys.has(key)) fields.delete(key); + }); Object.entries(modifiedFields).forEach(([k, v]) => { const crdtInfo = crdtFields.get(k); + if (!crdtInfo && v === undefined) { + fields.delete(k); + return; + } // Handle register updates specially - set value with client ID if (crdtInfo?.type === 'register') { @@ -924,23 +1087,59 @@ export function convexCollectionOptions( return deltas; }; - const applyYjsDelete = (mutations: CollectionMutation[]): Uint8Array[] => { - const deltas: Uint8Array[] = []; + interface DeleteRollbackAttempt { + documentId: string; + document: Y.Doc; + generation: number; + deletedAt: unknown; + accepted: boolean; + } - for (const mut of mutations) { - const document = String(mut.key); - const ydoc = docManager.get(document); - - if (ydoc) { - const delta = applyDeleteMarkerToDoc(ydoc); - deltas.push(delta); - } else { - const delta = createDeleteDelta(); - deltas.push(delta); + const rollbackDeleteAttempt = async (attempt: DeleteRollbackAttempt): Promise => { + if (stopped || attempt.generation !== connection.generation) return; + if (docManager.get(attempt.documentId) !== attempt.document) return; + const meta = attempt.document.getMap('_meta'); + const previousRepairEpoch = fullSyncRequired.get(attempt.documentId); + const repairEpoch = ++fullSyncEpoch; + fullSyncRequired.set(attempt.documentId, repairEpoch); + try { + await persistence.kv.set(fullSyncKey(attempt.documentId), repairEpoch); + } catch (error) { + if (fullSyncRequired.get(attempt.documentId) === repairEpoch) { + if (previousRepairEpoch === undefined) fullSyncRequired.delete(attempt.documentId); + else fullSyncRequired.set(attempt.documentId, previousRepairEpoch); } + throw error; } - - return deltas; + ensureGeneration(attempt.generation); + if ( + docManager.get(attempt.documentId) !== attempt.document || + meta.get('_deleted') !== true || + meta.get('_deletedAt') !== attempt.deletedAt + ) + return; + let restored = false; + attempt.document.transact(() => { + if (meta.get('_deleted') !== true || meta.get('_deletedAt') !== attempt.deletedAt) return; + meta.delete('_deleted'); + meta.delete('_deletedAt'); + restored = true; + }, YjsOrigin.Local); + if (!restored) return; + revisions.set(attempt.documentId, (revisions.get(attempt.documentId) ?? 0) + 1); + await docManager.flush([attempt.documentId]); + ensureGeneration(attempt.generation); + if ( + docManager.get(attempt.documentId) !== attempt.document || + isDocumentDeleted(docManager, attempt.documentId) + ) + return; + const restoredItem = serializeDocument(docManager, attempt.documentId, crdtFields); + if (!restoredItem) { + throw new Error(`Cannot serialize document ${attempt.documentId} for rollback`); + } + ops.upsert([restoredItem as DataType]); + logger.debug('Rolled back delete', { documentId: attempt.documentId }); }; return { @@ -950,10 +1149,26 @@ export function convexCollectionOptions( extensions, onInsert: async ({ transaction }: CollectionTransaction) => { + assertWritable(); + const mutationGeneration = connection.generation; + await persistenceReadyPromise; + ensureGeneration(mutationGeneration); + await serialize(async () => { + ensureGeneration(mutationGeneration); + for (const mutation of transaction.mutations) { + const id = String(mutation.key); + pendingCreates.add(id); + await persistence.kv.set(pendingCreateKey(id), true); + } + }); + ensureGeneration(mutationGeneration); const deltas = applyYjsInsert(transaction.mutations); try { await Promise.all([persistenceReadyPromise, optimisticReadyPromise]); + ensureGeneration(mutationGeneration); + await docManager.flush(transaction.mutations.map((mutation) => String(mutation.key))); + ensureGeneration(mutationGeneration); // Process mutations in parallel for better performance await Promise.all( @@ -962,17 +1177,21 @@ export function convexCollectionOptions( if (!delta || delta.length === 0) return; const document = String(mut.key); + revisions.set(document, (revisions.get(document) ?? 0) + 1); const materializedDoc = serializeDocument(docManager, document); if (!materializedDoc) { throw new Error(`Cannot serialize document ${document} for insert`); } + ensureGeneration(mutationGeneration); await convexClient.mutation(api.replicate, { document: document, bytes: delta.buffer, material: materializedDoc, type: 'insert', }); + ensureGeneration(mutationGeneration); + await acknowledgeCreate(document, mutationGeneration); }) ); } catch (error) { @@ -981,6 +1200,8 @@ export function convexCollectionOptions( }, onUpdate: async ({ transaction }: CollectionTransaction) => { + assertWritable(); + const mutationGeneration = connection.generation; const mutation = transaction.mutations[0]; const documentKey = String(mutation.key); @@ -988,18 +1209,45 @@ export function convexCollectionOptions( const isContentSync = !!metadata?.contentSync; const deltas = isContentSync ? null : applyYjsUpdate(transaction.mutations); + if (isContentSync) + revisions.set(documentKey, (revisions.get(documentKey) ?? 0) + 1); try { await Promise.all([persistenceReadyPromise, optimisticReadyPromise]); + ensureGeneration(mutationGeneration); + await docManager.flush(transaction.mutations.map((mutation) => String(mutation.key))); + ensureGeneration(mutationGeneration); if (isContentSync && metadata?.contentSync) { const { bytes, material } = metadata.contentSync; + const repairEpoch = fullSyncRequired.get(documentKey); + const revision = revisions.get(documentKey) ?? 0; + const document = docManager.get(documentKey); + let repairBytes = bytes; + let repairMaterial = material; + if (repairEpoch !== undefined && document) { + const fullDoc = serializeDocument(docManager, documentKey); + if (!fullDoc) { + throw new Error(`Cannot serialize document ${documentKey} for update`); + } + repairBytes = Y.encodeStateAsUpdateV2(document).buffer as ArrayBuffer; + repairMaterial = fullDoc; + } + ensureGeneration(mutationGeneration); await convexClient.mutation(api.replicate, { document: documentKey, - bytes, - material, + bytes: repairBytes, + material: repairMaterial, type: 'update', }); + ensureGeneration(mutationGeneration); + if (repairEpoch !== undefined) + await acknowledgeFullSync( + documentKey, + repairEpoch, + mutationGeneration, + revision + ); return; } @@ -1011,17 +1259,32 @@ export function convexCollectionOptions( if (!delta || delta.length === 0) return; const docId = String(mut.key); + const repairEpoch = fullSyncRequired.get(docId); + const revision = revisions.get(docId) ?? 0; + const document = docManager.get(docId); const fullDoc = serializeDocument(docManager, docId); if (!fullDoc) { throw new Error(`Cannot serialize document ${docId} for update`); } + ensureGeneration(mutationGeneration); await convexClient.mutation(api.replicate, { document: docId, - bytes: delta.buffer, + bytes: + repairEpoch !== undefined && document + ? (Y.encodeStateAsUpdateV2(document).buffer as ArrayBuffer) + : delta.buffer, material: fullDoc, type: 'update', }); + ensureGeneration(mutationGeneration); + if (repairEpoch !== undefined) + await acknowledgeFullSync( + docId, + repairEpoch, + mutationGeneration, + revision + ); }) ); } @@ -1031,85 +1294,80 @@ export function convexCollectionOptions( }, onDelete: async ({ transaction }: CollectionTransaction) => { + assertWritable(); + const mutationGeneration = connection.generation; + const rollbackAttempts: DeleteRollbackAttempt[] = []; try { await Promise.all([persistenceReadyPromise, optimisticReadyPromise]); + ensureGeneration(mutationGeneration); + + for (const mut of transaction.mutations) { + const documentId = String(mut.key); + const ydoc = docManager.get(documentId); + const repairEpoch = fullSyncRequired.get(documentId); + const delta = ydoc ? applyDeleteMarkerToDoc(ydoc) : createDeleteDelta(); + const rollbackAttempt = ydoc + ? { + documentId, + document: ydoc, + generation: mutationGeneration, + deletedAt: ydoc.getMap('_meta').get('_deletedAt'), + accepted: false, + } + : undefined; + if (rollbackAttempt) rollbackAttempts.push(rollbackAttempt); + revisions.set(documentId, (revisions.get(documentId) ?? 0) + 1); + const revision = revisions.get(documentId) ?? 0; + await docManager.flush([documentId]); + ensureGeneration(mutationGeneration); + + const original = mut.original; + if (original && Object.keys(original).length > 0) { + ops.delete([original as DataType]); + } - if (!txCoordinator) { - // Fallback to non-transactional delete if coordinator not ready - logger.warn('Transaction coordinator not initialized, using fallback delete'); - const deltas = applyYjsDelete(transaction.mutations); - const itemsToDelete = transaction.mutations - .map((mut) => mut.original) - .filter((item): item is DataType => item !== undefined && Object.keys(item).length > 0); - ops.delete(itemsToDelete); - - await Promise.all( - transaction.mutations.map(async (mut, i) => { - const delta = deltas[i]; - if (!delta || delta.length === 0) return; - - await convexClient.mutation(api.replicate, { - document: String(mut.key), - bytes: delta.buffer, - type: 'delete', - }); - }) - ); - return; + ensureGeneration(mutationGeneration); + await convexClient.mutation(api.replicate, { + document: documentId, + bytes: + repairEpoch !== undefined && ydoc + ? (Y.encodeStateAsUpdateV2(ydoc).buffer as ArrayBuffer) + : delta.buffer, + type: 'delete', + }); + if (rollbackAttempt) rollbackAttempt.accepted = true; + ensureGeneration(mutationGeneration); + if (repairEpoch !== undefined) + await acknowledgeFullSync( + documentId, + repairEpoch, + mutationGeneration, + revision + ); } - - // Use transaction coordinator for atomic delete with rollback support - // Map to store previousState for rollback (keyed by documentId) - const rollbackStates = new Map(); - - await txCoordinator.transaction(async (tx) => { - // Process each deletion in the transaction - for (const mut of transaction.mutations) { - const documentId = String(mut.key); - const ydoc = docManager.get(documentId); - - // Capture state for potential rollback - if (ydoc) { - const previousState = Y.encodeStateAsUpdateV2(ydoc); - rollbackStates.set(documentId, previousState); - } - - // Stage the delete in the transaction - tx.stageDelete(documentId); - - // Create delete delta - const delta = ydoc ? applyDeleteMarkerToDoc(ydoc) : createDeleteDelta(); - - // Attach previousState to the staged change for rollback - const changes = tx.getStagedChanges(); - const lastChange = changes[changes.length - 1]; - if (lastChange && rollbackStates.has(documentId)) { - // StagedChange interface allows previousState, safe to assign via mutable reference - (lastChange as StagedChange).previousState = rollbackStates.get(documentId); - } - - // Optimistically remove from TanStack DB - const original = mut.original; - if (original && Object.keys(original).length > 0) { - ops.delete([original as DataType]); - } - - // Call server mutation - if this fails, transaction will rollback - await convexClient.mutation(api.replicate, { - document: documentId, - bytes: delta.buffer, - type: 'delete', - }); - } - }); } catch (error) { - // Transaction already rolled back if error occurred + const rollbackErrors: unknown[] = []; + for (const attempt of [...rollbackAttempts].reverse()) { + if (attempt.accepted) continue; + try { + await serialize(() => rollbackDeleteAttempt(attempt)); + } catch (rollbackError) { + rollbackErrors.push(rollbackError); + } + } + if (rollbackErrors.length > 0) { + const cause = error instanceof Error ? error.message : String(error); + throw new AggregateError( + [error, ...rollbackErrors], + `Delete failed and local rollback could not be persisted: ${cause}` + ); + } handleMutationError(error); } }, sync: { - rowUpdateMode: 'partial', + rowUpdateMode: 'full', sync: (params: any) => { const { markReady, collection: collectionInstance } = params; @@ -1125,13 +1383,20 @@ export function convexCollectionOptions( const ssrDocuments = material?.documents; type CrdtRecord = Record; const ssrCrdt = material?.crdt as CrdtRecord | undefined; - const ssrCursor = material?.cursor; const docs: DataType[] = ssrDocuments ? [...ssrDocuments] : []; - (async () => { + localReady = (async () => { try { const existingDocIds = await persistence.listDocuments(collection); + if (stopped) return; for (const docId of existingDocIds) { + if (await persistence.kv.get(pendingCreateKey(docId))) pendingCreates.add(docId); + const repairEpoch = await persistence.kv.get(fullSyncKey(docId)); + if (typeof repairEpoch === 'number' && Number.isFinite(repairEpoch)) { + fullSyncRequired.set(docId, repairEpoch); + fullSyncEpoch = Math.max(fullSyncEpoch, repairEpoch); + } + if (stopped) return; docManager.getOrCreate(docId); } @@ -1139,69 +1404,25 @@ export function convexCollectionOptions( return persistence.createDocPersistence(`${collection}:${document}`, ydoc); }); await Promise.all(docPromises); + if (stopped) return; resolvePersistenceReady?.(); const clientId = await getClientId(persistence.kv); + if (stopped) return; updateContext(collection, { clientId }); ops = createReplicateOps(params); - // Initialize transaction coordinator with document-level apply/revert callbacks - txCoordinator = createTransactionCoordinator({ - async onApply(change: StagedChange) { - // Changes are already applied optimistically before server call - // This callback is for any post-commit bookkeeping - logger.debug('Transaction change applied', { - type: change.type, - documentId: change.documentId, - }); - }, - async onRevert(change: StagedChange) { - // Delete rollback - restore document - if (change.type === 'delete' && change.previousState) { - const update = new Uint8Array(change.previousState); - docManager.applyUpdate(change.documentId, update, YjsOrigin.Local); - const restoredItem = serializeDocument(docManager, change.documentId, crdtFields); - if (!restoredItem) { - throw new Error(`Cannot serialize document ${change.documentId} for rollback`); - } - ops.insert([restoredItem as DataType]); - logger.debug('Rolled back delete', { documentId: change.documentId }); - return; - } - - // Update rollback - restore previous state - if (change.type === 'update' && change.previousState) { - const update = new Uint8Array(change.previousState); - docManager.applyUpdate(change.documentId, update, YjsOrigin.Local); - const restoredItem = serializeDocument(docManager, change.documentId, crdtFields); - if (!restoredItem) { - throw new Error(`Cannot serialize document ${change.documentId} for rollback`); - } - ops.upsert([restoredItem as DataType]); - logger.debug('Rolled back update', { documentId: change.documentId }); - return; - } - - // Insert rollback - delete (no-op, document wasn't persisted) - if (change.type === 'insert') { - ops.delete([{ id: change.documentId } as DataType]); - } - }, - }); - resolveOptimisticReady?.(); if (ssrCrdt) { for (const [docId, state] of Object.entries(ssrCrdt)) { const update = new Uint8Array(state.bytes); - docManager.applyUpdate(docId, update, YjsOrigin.Server); + await docManager.applyRemote(docId, update); } } - await recover(); - const docIds = docManager.documents(); if (docIds.length > 0) { const items = extractAllDocuments(docManager, crdtFields) as DataType[]; @@ -1213,215 +1434,87 @@ export function convexCollectionOptions( markReady(); getContext(collection).resolve?.(); - const persistedCursor = await seqService.load(collection); - let cursor = ssrCursor ?? persistedCursor; - - if (cursor > 0 && docManager.documents().length === 0) { - cursor = 0; - persistence.kv.set(`cursor:${collection}`, 0); - } - - // Signal that sync is ready (no actor system needed - sync manager is self-contained) getContext(collection).resolveActorReady?.(); - - // Returns { item, isNew, isDelete } for batching, null if no action needed - type ChangeResult = { - item: DataType; - isNew: boolean; - isDelete: boolean; - } | null; - - // Clean up presence provider when a document is deleted - const cleanupDocumentPresence = (documentId: string) => { - const provider = presenceProviders.get(documentId); - if (provider) { - provider.destroy(); - presenceProviders.delete(documentId); - } - documentHandles.delete(documentId); - }; - - const handleSnapshotChange = withErrorIsolation( - (bytes: ArrayBuffer, document: string, exists: boolean): ChangeResult => { - const hadLocally = docManager.has(document); - - if (!exists && hadLocally) { - const itemBefore = serializeDocument(docManager, document, crdtFields); - docManager.delete(document); - cleanupDocumentPresence(document); - if (itemBefore) { - return { item: itemBefore as DataType, isNew: false, isDelete: true }; - } - return null; - } - - if (!exists && !hadLocally) { - return null; - } - - // Apply update - use hadLocally for existence check (avoid double serialization) - const update = new Uint8Array(bytes); - docManager.applyUpdate(document, update, YjsOrigin.Server); - const itemAfter = serializeDocument(docManager, document, crdtFields); - - if (itemAfter) { - return { item: itemAfter as DataType, isNew: !hadLocally, isDelete: false }; - } else if (hadLocally) { - // Serialization failed - log warning but don't return item - logger.warn('Document serialization returned null after snapshot update', { - document, - collection, - hadFieldsAfter: !!docManager.getFields(document), - }); - } - return null; - }, - (error, _bytes, document) => { - logger.error('Error processing snapshot change', { document, collection, error }); - return null; - } - ); - - const handleDeltaChange = withErrorIsolation( - (bytes: ArrayBuffer, document: string | undefined, exists: boolean): ChangeResult => { - if (!document) { - return null; - } - - const hadLocally = docManager.has(document); - - if (!exists && hadLocally) { - const itemBefore = serializeDocument(docManager, document, crdtFields); - docManager.delete(document); - cleanupDocumentPresence(document); - if (itemBefore) { - return { item: itemBefore as DataType, isNew: false, isDelete: true }; - } - return null; - } - - if (!exists && !hadLocally) { - return null; - } - - // Apply update - use hadLocally for existence check (avoid double serialization) - const update = new Uint8Array(bytes); - docManager.applyUpdate(document, update, YjsOrigin.Server); - const itemAfter = serializeDocument(docManager, document, crdtFields); - - if (itemAfter) { - return { item: itemAfter as DataType, isNew: !hadLocally, isDelete: false }; - } else if (hadLocally) { - // Serialization failed - log warning but don't return item - logger.warn('Document serialization returned null after delta update', { - document, - collection, - hadFieldsAfter: !!docManager.getFields(document), - }); - } - return null; - }, - (error, _bytes, document) => { - logger.error('Error processing delta change', { document, collection, error }); - return null; - } - ); - - let lastProcessedSeq = cursor; - - const handleSubscriptionUpdate = async (response: any) => { - if (!response || !Array.isArray(response.changes)) { - return; - } - - const { changes, seq: newSeq } = response; - - // Skip if we've already processed up to this seq — prevents - // re-enqueuing presence marks on subscription re-fires (e.g. reconnect) - if (newSeq !== undefined && newSeq <= lastProcessedSeq) { - return; - } - - const syncedDocuments = new Set(); - - // Process all changes and collect results for batching - const toInsert: DataType[] = []; - const toUpsert: DataType[] = []; - const toDelete: DataType[] = []; - - for (const change of changes) { - const { type, bytes, document, exists } = change; - if (!bytes || !document) { - continue; - } - - syncedDocuments.add(document); - - const result = - type === 'snapshot' - ? handleSnapshotChange(bytes, document, exists ?? true) - : handleDeltaChange(bytes, document, exists ?? true); - - if (result) { - if (result.isDelete) { - toDelete.push(result.item); - } else if (result.isNew) { - toInsert.push(result.item); - } else { - toUpsert.push(result.item); - } - } - } - - // Batch ops calls - single transaction instead of N separate calls - if (toDelete.length > 0) ops.delete(toDelete); - if (toInsert.length > 0) ops.insert(toInsert); - if (toUpsert.length > 0) ops.upsert(toUpsert); - - if (newSeq !== undefined) { - lastProcessedSeq = newSeq; - persistence.kv.set(`cursor:${collection}`, newSeq); - - // Mark presence for synced documents using background sync queue - // The queue provides retry with exponential backoff on transient failures - for (const document of syncedDocuments) { - syncQueue.enqueue(`presence:${document}`, async () => { - const vector = docManager.encodeStateVector(document); - await convexClient.mutation(api.presence, { - document, - client: clientId, - action: 'mark', - seq: newSeq, - vector: vector.buffer as ArrayBuffer, - }); - }); - } - - // Resubscribe with the advanced cursor so the reactive query - // only returns genuinely new deltas, not ones we've already processed. - // Without this, reconnection or any table change causes the query - // to re-fire with ALL deltas since the original cursor. + startNetwork = async () => { + const token = connection.generation; + let cursor = await seqService.load(collection); + if (stopped || !connection.active || token !== connection.generation) return; + // Recovery never delays publishing persisted local rows or starting discovery. + void recover(); + const subscribe = () => { + if (!connection.active || token !== connection.generation) return; subscription?.(); subscription = convexClient.onUpdate( api.delta, - { seq: newSeq, limit: 1000 }, + { seq: cursor, limit: 1000 }, (response: any) => { - handleSubscriptionUpdate(response); + void serialize(async () => { + if (!connection.active || token !== connection.generation) return; + if (!response || !Array.isArray(response.changes) || response.seq <= cursor) + return; + const documents = new Set(); + for (const change of response.changes) { + if (change.seq <= cursor) continue; + if (!change.document) + throw new Error('Stream change is missing its document'); + documents.add(change.document); + revisions.set(change.document, (revisions.get(change.document) ?? 0) + 1); + if (change.exists === false) { + await removeDocument(change.document); + } else { + if (!change.bytes) throw new Error('Stream change is missing its update'); + await docManager.applyRemote( + change.document, + new Uint8Array(change.bytes) + ); + } + } + const acknowledgements = [...documents].map((document) => ({ + document, + vector: docManager.encodeStateVector(document), + })); + await docManager.flush(documents); + // Disconnect drains this accepted local work. Publish its durable + // rows even if it cancels the subsequent checkpoint/acknowledgement. + publishDocuments(documents); + if (!connection.active || token !== connection.generation) return; + await seqService.save(collection, response.seq); + if (!connection.active || token !== connection.generation) return; + cursor = response.seq; + for (const { document, vector } of acknowledgements) { + durableVectors.set(document, vector.buffer as ArrayBuffer); + syncQueue.enqueue(`presence:${document}`, async () => { + if (!connection.active || token !== connection.generation) return; + await convexClient.mutation(api.presence, { + document, + client: clientId, + action: 'mark', + seq: response.seq, + vector: vector.buffer as ArrayBuffer, + }); + }); + } + subscribe(); + }).catch((error) => { + logger.error('Stream persistence failed; retrying from durable cursor', { + collection, + error, + }); + if (connection.active && token === connection.generation) { + clearTimeout(retryTimer); + retryTimer = setTimeout(subscribe, 1000); + } + }); } ); - } + }; + subscribe(); + if (sessionSubscribers.size) initSessionSubscription(); }; - - // Subscribe to the delta stream. Convex handles reactivity — - // when new deltas are written to the server, the query re-evaluates - // and the callback fires with the updated result automatically. - subscription = convexClient.onUpdate( - api.delta, - { seq: cursor, limit: 1000 }, - (response: any) => { - handleSubscriptionUpdate(response); - } - ); + if (desiredConnection && !stopped) { + connection.connect(); + void startNetwork(); + } // Reconnection handling: when browser comes back online, resync local state if (typeof globalThis.window !== 'undefined') { @@ -1431,7 +1524,7 @@ export function convexCollectionOptions( logger.debug('Network offline detected', { collection }); }; const handleOnline = () => { - if (wasOffline) { + if (wasOffline && connection.active) { logger.info('Network online restored, running recovery sync', { collection }); wasOffline = false; recover(true).catch((error: Error) => { @@ -1470,6 +1563,9 @@ export function convexCollectionOptions( return { material: docs, cleanup: () => { + stopped = true; + clearTimeout(retryTimer); + connection.disconnect(); // Clean up reconnection listeners if stored in context if (hasContext(collection)) { const ctx = getContext(collection); @@ -1483,6 +1579,9 @@ export function convexCollectionOptions( provider.destroy(); } presenceProviders.clear(); + for (const providers of editorPresenceProviders.values()) + for (const provider of providers) provider.destroy(); + editorPresenceProviders.clear(); documentHandles.clear(); deleteContext(collection); docPersistence?.destroy(); @@ -1525,7 +1624,7 @@ export type ConvexCollection = Collection< ConvexCollectionExtensions; /** Options for collection.create() */ -export interface CreateCollectionOptions { +export interface CreateCollectionOptions extends ConnectionOptions { schema: VersionedSchema; persistence: () => Promise; config: () => { @@ -1592,6 +1691,9 @@ function createVersionedCollection( api: userConfig.api, getKey: userConfig.getKey, user: userConfig.user, + networkMode: options.networkMode, + readOnly: options.readOnly, + requestArgs: options.requestArgs, } as LazyCollectionConfig; if (isPaginatedMaterial(mat)) { diff --git a/packages/replicate/src/client/documents.ts b/packages/replicate/src/client/documents.ts index 19b6855..acb74ad 100644 --- a/packages/replicate/src/client/documents.ts +++ b/packages/replicate/src/client/documents.ts @@ -12,10 +12,13 @@ export interface DocumentManager { getOrCreate(id: string): Y.Doc; has(id: string): boolean; delete(id: string): void; + remove(id: string): Promise; + flush(ids?: Iterable): Promise; getFields(id: string): Y.Map | null; getFragment(id: string, field: string): Y.XmlFragment | null; + applyRemote(id: string, update: Uint8Array): Promise; applyUpdate(id: string, update: Uint8Array, origin?: string): void; encodeState(id: string): Uint8Array; encodeStateVector(id: string): Uint8Array; @@ -72,6 +75,26 @@ export function createDocumentManager(collection: string): DocumentManager { } }, + async remove(id: string): Promise { + const provider = persistence.get(id); + if (provider && !provider.clear) { + throw new Error('PersistenceProvider.clear() is required for durable removal'); + } + await provider?.clear?.(); + this.delete(id); + }, + + async flush(ids: Iterable = persistence.keys()): Promise { + for (const id of ids) { + const provider = persistence.get(id); + if (!provider) continue; + await provider.whenSynced; + if (!provider.flush) + throw new Error('PersistenceProvider.flush() is required for durable progress'); + await provider.flush(); + } + }, + getFields(id: string): Y.Map | null { const doc = docs.get(id); return doc ? doc.getMap('fields') : null; @@ -89,6 +112,20 @@ export function createDocumentManager(collection: string): DocumentManager { return null; }, + async applyRemote(id: string, update: Uint8Array): Promise { + const doc = this.getOrCreate(id); + const provider = persistence.get(id); + if (provider) { + if (!provider.storeUpdate) + throw new Error( + 'PersistenceProvider.storeUpdate() is required for durable remote updates' + ); + await provider.whenSynced; + await provider.storeUpdate(update); + } + Y.applyUpdateV2(doc, update, 'replicate-persisted'); + }, + applyUpdate(id: string, update: Uint8Array, origin?: string): void { const doc = this.getOrCreate(id); Y.applyUpdateV2(doc, update, origin); diff --git a/packages/replicate/src/client/persistence/custom.ts b/packages/replicate/src/client/persistence/custom.ts index 5fbc7fd..829f23c 100644 --- a/packages/replicate/src/client/persistence/custom.ts +++ b/packages/replicate/src/client/persistence/custom.ts @@ -1,3 +1,4 @@ +import { PersistenceWrites } from './writes.js'; import * as Y from 'yjs'; import type { StorageAdapter, Persistence, PersistenceProvider, KeyValueStore } from './types.js'; @@ -26,6 +27,8 @@ class AdapterKeyValueStore implements KeyValueStore { class AdapterPersistenceProvider implements PersistenceProvider { private updateHandler: (update: Uint8Array, origin: unknown) => void; private updateCounter = 0; + private writes = new PersistenceWrites(); + private acceptingWrites = true; readonly whenSynced: Promise; constructor( @@ -36,8 +39,11 @@ class AdapterPersistenceProvider implements PersistenceProvider { this.whenSynced = this.loadState(); this.updateHandler = (update: Uint8Array, origin: unknown) => { - if (origin !== 'custom') { - void this.saveUpdate(update); + if (this.acceptingWrites && origin !== 'custom' && origin !== 'replicate-persisted') { + this.writes.add(async () => { + await this.whenSynced; + await this.saveUpdate(update); + }); } }; this.ydoc.on('update', this.updateHandler); @@ -70,7 +76,32 @@ class AdapterPersistenceProvider implements PersistenceProvider { await this.adapter.set(`${UPDATE_PREFIX}${this.collection}:${paddedCounter}`, update); } + async storeUpdate(update: Uint8Array): Promise { + if (!this.acceptingWrites) throw new Error('Persistence provider is closed'); + const bytes = Y.convertUpdateFormatV2ToV1(update); + this.writes.add(async () => { + await this.whenSynced; + await this.saveUpdate(bytes); + }); + await this.flush(); + } + + async flush(): Promise { + await this.whenSynced; + await this.writes.flush(); + } + + async clear(): Promise { + this.destroy(); + await this.flush(); + await this.adapter.delete(`${SNAPSHOT_PREFIX}${this.collection}`); + for (const key of await this.adapter.keys(`${UPDATE_PREFIX}${this.collection}:`)) { + await this.adapter.delete(key); + } + } + destroy(): void { + this.acceptingWrites = false; this.ydoc.off('update', this.updateHandler); } } diff --git a/packages/replicate/src/client/persistence/encrypted/web.ts b/packages/replicate/src/client/persistence/encrypted/web.ts index c880e4a..640914e 100644 --- a/packages/replicate/src/client/persistence/encrypted/web.ts +++ b/packages/replicate/src/client/persistence/encrypted/web.ts @@ -1,3 +1,4 @@ +import { PersistenceWrites } from '../writes.js'; import * as Y from 'yjs'; import type { Persistence, PersistenceProvider, KeyValueStore } from '../types.js'; import type { WebEncryptionConfig, EncryptionPersistence, EncryptionState } from './types.js'; @@ -80,25 +81,24 @@ class EncryptedKeyValueStore implements KeyValueStore { class EncryptedPersistenceProvider implements PersistenceProvider { private updateHandler: (update: Uint8Array, origin: unknown) => void; - private pendingWrites: Promise[] = []; + private writes = new PersistenceWrites(); + private acceptingWrites = true; readonly whenSynced: Promise; constructor( private innerStorage: Persistence, private collection: string, private ydoc: Y.Doc, - private encryptionKey: CryptoKey + private encryptionKey: CryptoKey, + private index: (name: string, remove?: boolean) => Promise ) { this.whenSynced = this.loadState(); this.updateHandler = (update: Uint8Array, origin: unknown) => { - if (origin !== 'encrypted-load') { - const writePromise = this.saveUpdate(update).catch((err: Error) => { - console.error('[EncryptedPersistence] Save failed:', err); - }); - this.pendingWrites.push(writePromise); - writePromise.finally(() => { - this.pendingWrites = this.pendingWrites.filter((p) => p !== writePromise); + if (this.acceptingWrites && origin !== 'encrypted-load' && origin !== 'replicate-persisted') { + this.writes.add(async () => { + await this.whenSynced; + await this.saveUpdate(update); }); } }; @@ -126,6 +126,7 @@ class EncryptedPersistenceProvider implements PersistenceProvider { private async saveUpdate(update: Uint8Array): Promise { const deltasKey = `${DOC_PREFIX}${this.collection}:deltas`; + await this.index(this.collection); const encrypted = await encrypt(this.encryptionKey, update); const existingDeltas = (await this.innerStorage.kv.get(deltasKey)) ?? []; @@ -133,15 +134,15 @@ class EncryptedPersistenceProvider implements PersistenceProvider { await this.innerStorage.kv.set(deltasKey, existingDeltas); if (existingDeltas.length >= 50) { - await this.compact(); + await this.compact(update); } } - private async compact(): Promise { + private async compact(update: Uint8Array): Promise { const snapshotKey = `${DOC_PREFIX}${this.collection}:snapshot`; const deltasKey = `${DOC_PREFIX}${this.collection}:deltas`; - const snapshot = Y.encodeStateAsUpdate(this.ydoc); + const snapshot = Y.mergeUpdates([Y.encodeStateAsUpdate(this.ydoc), update]); const encrypted = await encrypt(this.encryptionKey, snapshot); await this.innerStorage.kv.set(snapshotKey, Array.from(encrypted)); @@ -149,10 +150,29 @@ class EncryptedPersistenceProvider implements PersistenceProvider { } async flush(): Promise { - await Promise.all(this.pendingWrites); + await this.whenSynced; + await this.writes.flush(); + } + + async storeUpdate(update: Uint8Array): Promise { + if (!this.acceptingWrites) throw new Error('Persistence provider is closed'); + this.writes.add(async () => { + await this.whenSynced; + await this.saveUpdate(Y.convertUpdateFormatV2ToV1(update)); + }); + await this.flush(); + } + + async clear(): Promise { + this.destroy(); + await this.flush(); + await this.innerStorage.kv.del(`${DOC_PREFIX}${this.collection}:snapshot`); + await this.innerStorage.kv.del(`${DOC_PREFIX}${this.collection}:deltas`); + await this.index(this.collection, true); } destroy(): void { + this.acceptingWrites = false; this.ydoc.off('update', this.updateHandler); } } @@ -280,6 +300,19 @@ export async function createWebEncryptionPersistence( throw new Error('No unlock method configured'); }; + const indexWrites = new PersistenceWrites(); + const index = async (name: string, remove = false) => { + indexWrites.add(async () => { + const separator = name.indexOf(':'); + const key = `enc:documents:${name.slice(0, separator)}`; + const id = name.slice(separator + 1); + const ids = new Set((await storage.kv.get(key)) ?? []); + if (remove) ids.delete(id); + else ids.add(id); + await storage.kv.set(key, [...ids]); + }); + await indexWrites.flush(); + }; const encryptedKv = new EncryptedKeyValueStore(storage.kv, () => encryptionKey); return { @@ -306,12 +339,13 @@ export async function createWebEncryptionPersistence( if (!encryptionKey) { throw new Error('Encryption locked - call unlock() first'); } - return new EncryptedPersistenceProvider(storage, collection, ydoc, encryptionKey); + return new EncryptedPersistenceProvider(storage, collection, ydoc, encryptionKey, index); }, async listDocuments(prefix: string): Promise { - const keys = await storage.listDocuments(prefix); - return keys; + const existing = await storage.listDocuments(prefix); + const indexed = (await storage.kv.get(`enc:documents:${prefix}`)) ?? []; + return [...new Set([...existing, ...indexed])]; }, kv: encryptedKv, diff --git a/packages/replicate/src/client/persistence/memory.ts b/packages/replicate/src/client/persistence/memory.ts index 2b02e94..dfb659c 100644 --- a/packages/replicate/src/client/persistence/memory.ts +++ b/packages/replicate/src/client/persistence/memory.ts @@ -32,9 +32,18 @@ class MemoryKeyValueStore implements KeyValueStore { */ class MemoryPersistenceProvider implements PersistenceProvider { readonly whenSynced = Promise.resolve(); + private acceptingWrites = true; + + async storeUpdate(_update: Uint8Array): Promise { + if (!this.acceptingWrites) throw new Error('Persistence provider is closed'); + } + async flush(): Promise {} + async clear(): Promise { + this.destroy(); + } destroy(): void { - // No resources to clean up + this.acceptingWrites = false; } } diff --git a/packages/replicate/src/client/persistence/sqlite/native.ts b/packages/replicate/src/client/persistence/sqlite/native.ts index 5b1ba8e..93ecc24 100644 --- a/packages/replicate/src/client/persistence/sqlite/native.ts +++ b/packages/replicate/src/client/persistence/sqlite/native.ts @@ -1,4 +1,9 @@ -import { initSchema, createPersistenceFromExecutor, type Executor } from './schema.js'; +import { + initSchema, + createPersistenceFromExecutor, + type Executor, + type SqliteStatement, +} from './schema.js'; import type { Persistence } from '../types.js'; interface OPSQLiteDatabase { @@ -7,15 +12,52 @@ interface OPSQLiteDatabase { } class OPSqliteExecutor implements Executor { + private tail: Promise = Promise.resolve(); + private closing = false; + private closePromise: Promise | null = null; + constructor(private db: OPSQLiteDatabase) {} - async execute(sql: string, params?: unknown[]): Promise<{ rows: Record[] }> { - const result = await this.db.execute(sql, params); - return { rows: result.rows || [] }; + private enqueue(operation: () => Promise): Promise { + if (this.closing) return Promise.reject(new Error('SQLite database is closing')); + const result = this.tail.catch(() => {}).then(operation); + this.tail = result; + return result; + } + + execute(sql: string, params?: unknown[]): Promise<{ rows: Record[] }> { + return this.enqueue(async () => { + const result = await this.db.execute(sql, params); + return { rows: result.rows || [] }; + }); + } + + transaction(statements: readonly SqliteStatement[]): Promise { + return this.enqueue(async () => { + await this.db.execute('BEGIN IMMEDIATE'); + try { + for (const statement of statements) { + await this.db.execute(statement.sql, statement.params); + } + await this.db.execute('COMMIT'); + } catch (error) { + try { + await this.db.execute('ROLLBACK'); + } catch { + // Preserve the statement failure that caused the rollback. + } + throw error; + } + }); } - close(): void { - this.db.close(); + async close(): Promise { + if (!this.closePromise) { + this.closing = true; + this.closePromise = this.tail.catch(() => {}).then(() => this.db.close()); + this.tail = this.closePromise; + } + await this.closePromise; } } diff --git a/packages/replicate/src/client/persistence/sqlite/preload.ts b/packages/replicate/src/client/persistence/sqlite/preload.ts index 7c38191..cf8126a 100644 --- a/packages/replicate/src/client/persistence/sqlite/preload.ts +++ b/packages/replicate/src/client/persistence/sqlite/preload.ts @@ -30,8 +30,8 @@ export const WASM_URL = `${CDN_BASE}/dist/wa-sqlite-async.wasm`; * * ``` */ -export function preloadLinks(): string { - return ``; +export function preloadLinks(assetBase = CDN_BASE): string { + return ``; } /** @@ -49,16 +49,20 @@ export function preloadLinks(): string { * * @returns A promise resolving to the compiled WebAssembly.Module */ -let _compiled: Promise | null = null; +const compiled = new Map>(); -export function compileWasmModule(): Promise { +export function compileWasmModule(assetBase = CDN_BASE): Promise { if (typeof window === 'undefined') { return Promise.reject(new Error('compileWasmModule is browser-only')); } - if (!_compiled) { - _compiled = WebAssembly.compileStreaming(fetch(WASM_URL)); + const url = `${assetBase.replace(/\/$/, '')}/dist/wa-sqlite-async.wasm`; + let result = compiled.get(url); + if (!result) { + result = WebAssembly.compileStreaming(fetch(url)); + compiled.set(url, result); + void result.catch(() => compiled.delete(url)); } - return _compiled; + return result; } /** @@ -73,7 +77,7 @@ export function compileWasmModule(): Promise { * injectPreloadLinks(); * ``` */ -export function injectPreloadLinks(): void { +export function injectPreloadLinks(assetBase = CDN_BASE): void { if (typeof document === 'undefined') return; const link = document.createElement('link'); @@ -81,6 +85,6 @@ export function injectPreloadLinks(): void { link.as = 'fetch'; link.type = 'application/wasm'; link.crossOrigin = 'anonymous'; - link.href = WASM_URL; + link.href = `${assetBase.replace(/\/$/, '')}/dist/wa-sqlite-async.wasm`; document.head.appendChild(link); } diff --git a/packages/replicate/src/client/persistence/sqlite/schema.ts b/packages/replicate/src/client/persistence/sqlite/schema.ts index dc44a56..4f6381e 100644 --- a/packages/replicate/src/client/persistence/sqlite/schema.ts +++ b/packages/replicate/src/client/persistence/sqlite/schema.ts @@ -1,3 +1,4 @@ +import { PersistenceWrites } from '../writes.js'; import * as Y from 'yjs'; import type { Persistence, @@ -8,7 +9,13 @@ import type { export interface Executor { execute(sql: string, params?: unknown[]): Promise<{ rows: Record[] }>; - close(): void; + transaction(statements: readonly SqliteStatement[]): Promise; + close(): void | Promise; +} + +export interface SqliteStatement { + sql: string; + params?: unknown[]; } export async function initSchema(executor: Executor): Promise { @@ -42,15 +49,20 @@ export async function initSchema(executor: Executor): Promise { } class SqliteKeyValueStore implements KeyValueStore { - constructor(private executor: Executor) {} + constructor( + private executor: Executor, + private assertOpen: () => void + ) {} async get(key: string): Promise { + this.assertOpen(); const result = await this.executor.execute('SELECT value FROM kv WHERE key = ?', [key]); if (result.rows.length === 0) return undefined; return JSON.parse(result.rows[0].value as string) as T; } async set(key: string, value: T): Promise { + this.assertOpen(); await this.executor.execute('INSERT OR REPLACE INTO kv (key, value) VALUES (?, ?)', [ key, JSON.stringify(value), @@ -58,6 +70,7 @@ class SqliteKeyValueStore implements KeyValueStore { } async del(key: string): Promise { + this.assertOpen(); await this.executor.execute('DELETE FROM kv WHERE key = ?', [key]); } } @@ -66,23 +79,30 @@ class SqliteKeyValueStore implements KeyValueStore { * Adapter that wraps Executor to provide MigrationDatabase interface. */ class SqliteMigrationDatabase implements MigrationDatabase { - constructor(private executor: Executor) {} + constructor( + private executor: Executor, + private assertOpen: () => void + ) {} async run(sql: string, params?: unknown[]): Promise { + this.assertOpen(); await this.executor.execute(sql, params); } async exec(sql: string): Promise { + this.assertOpen(); await this.executor.execute(sql); } async get(sql: string, params?: unknown[]): Promise { + this.assertOpen(); const result = await this.executor.execute(sql, params); if (result.rows.length === 0) return undefined; return result.rows[0] as T; } async all(sql: string, params?: unknown[]): Promise { + this.assertOpen(); const result = await this.executor.execute(sql, params); return result.rows as T[]; } @@ -90,40 +110,50 @@ class SqliteMigrationDatabase implements MigrationDatabase { class SqlitePersistenceProvider implements PersistenceProvider { private updateHandler: (update: Uint8Array, origin: unknown) => void; - private pendingWrites: Promise[] = []; - private lastError: Error | null = null; + private writes = new PersistenceWrites(); + private acceptingWrites = true; readonly whenSynced: Promise; constructor( private executor: Executor, private collection: string, - private ydoc: Y.Doc, - private onError?: (error: Error) => void + private ydoc: Y.Doc ) { this.whenSynced = this.loadState(); this.updateHandler = (update: Uint8Array, origin: unknown) => { - if (origin !== 'sqlite') { - const writePromise = this.saveUpdate(update).catch((error: Error) => { - this.lastError = error; - this.onError?.(error); - }); - this.pendingWrites.push(writePromise); - writePromise.finally(() => { - this.pendingWrites = this.pendingWrites.filter((p) => p !== writePromise); + if (this.acceptingWrites && origin !== 'sqlite' && origin !== 'replicate-persisted') { + this.writes.add(async () => { + await this.whenSynced; + await this.saveUpdate(update); }); } }; this.ydoc.on('update', this.updateHandler); } + async storeUpdate(update: Uint8Array): Promise { + if (!this.acceptingWrites) throw new Error('Persistence provider is closed'); + const bytes = Y.convertUpdateFormatV2ToV1(update); + this.writes.add(async () => { + await this.whenSynced; + await this.saveUpdate(bytes); + }); + await this.flush(); + } + async flush(): Promise { - await Promise.all(this.pendingWrites); - if (this.lastError) { - const error = this.lastError; - this.lastError = null; - throw error; - } + await this.whenSynced; + await this.writes.flush(); + } + + async clear(): Promise { + this.destroy(); + await this.flush(); + await this.executor.transaction([ + { sql: 'DELETE FROM snapshots WHERE collection = ?', params: [this.collection] }, + { sql: 'DELETE FROM deltas WHERE collection = ?', params: [this.collection] }, + ]); } private async loadState(): Promise { @@ -158,15 +188,59 @@ class SqlitePersistenceProvider implements PersistenceProvider { } destroy(): void { + this.acceptingWrites = false; this.ydoc.off('update', this.updateHandler); } } export function createPersistenceFromExecutor(executor: Executor): Persistence { + const providers = new Set(); + let closing = false; + let closePromise: Promise | null = null; + const assertOpen = () => { + if (closing) throw new Error('SQLite persistence is closing'); + }; + const close = async () => { + const active = [...providers]; + for (const provider of active) provider.destroy(); + const failures: unknown[] = []; + for (const result of await Promise.allSettled(active.map((provider) => provider.flush?.()))) { + if (result.status === 'rejected') failures.push(result.reason); + } + try { + await executor.close(); + } catch (error) { + failures.push(error); + } finally { + providers.clear(); + } + if (failures.length === 1) throw failures[0]; + if (failures.length > 1) throw new AggregateError(failures, 'Persistence close failed'); + }; return { - createDocPersistence: (collection: string, ydoc: Y.Doc) => - new SqlitePersistenceProvider(executor, collection, ydoc), + close() { + if (!closePromise) { + closing = true; + closePromise = close(); + } + return closePromise; + }, + createDocPersistence: (collection: string, ydoc: Y.Doc) => { + assertOpen(); + const provider = new SqlitePersistenceProvider(executor, collection, ydoc); + providers.add(provider); + const detach = provider.destroy.bind(provider); + provider.destroy = () => { + detach(); + void provider.flush().then( + () => providers.delete(provider), + () => {} + ); + }; + return provider; + }, async listDocuments(prefix: string): Promise { + assertOpen(); const result = await executor.execute( `SELECT DISTINCT collection FROM ( SELECT collection FROM snapshots WHERE collection LIKE ? @@ -181,7 +255,7 @@ export function createPersistenceFromExecutor(executor: Executor): Persistence { return parts.slice(1).join(':'); }); }, - kv: new SqliteKeyValueStore(executor), - db: new SqliteMigrationDatabase(executor), + kv: new SqliteKeyValueStore(executor, assertOpen), + db: new SqliteMigrationDatabase(executor, assertOpen), }; } diff --git a/packages/replicate/src/client/persistence/sqlite/web.ts b/packages/replicate/src/client/persistence/sqlite/web.ts index 61d3cd4..328591a 100644 --- a/packages/replicate/src/client/persistence/sqlite/web.ts +++ b/packages/replicate/src/client/persistence/sqlite/web.ts @@ -1,18 +1,21 @@ -import { createPersistenceFromExecutor, type Executor } from './schema.js'; +import { createPersistenceFromExecutor, type Executor, type SqliteStatement } from './schema.js'; import type { Persistence } from '../types.js'; -import { compileWasmModule } from './preload.js'; +import { compileWasmModule, CDN_BASE } from './preload.js'; const INIT = 0; const EXECUTE = 1; const CLOSE = 2; +const TRANSACTION = 4; interface Request { id: number; type: number; name?: string; wasmModule?: WebAssembly.Module; + assetBase?: string; sql?: string; params?: unknown[]; + statements?: readonly SqliteStatement[]; } interface Response { @@ -32,6 +35,7 @@ class WorkerExecutor implements Executor { private nextId = 0; private pending = new Map(); private terminated = false; + private closePromise: Promise | null = null; constructor(worker: Worker) { this.worker = worker; @@ -63,6 +67,7 @@ class WorkerExecutor implements Executor { private rejectAllPending(error: Error): void { this.terminated = true; + this.worker.terminate(); for (const [, handler] of this.pending) { handler.reject(error); } @@ -71,18 +76,23 @@ class WorkerExecutor implements Executor { private send(type: number, payload: Partial = {}): Promise[]> { return new Promise((resolve, reject) => { - if (this.terminated) { - reject(new Error('Worker has been terminated')); + if (this.terminated || (this.closePromise && type !== CLOSE)) { + reject(new Error(this.terminated ? 'Worker has been terminated' : 'Worker is closing')); return; } const id = this.nextId++; this.pending.set(id, { resolve, reject }); - this.worker.postMessage({ id, type, ...payload } satisfies Request); + try { + this.worker.postMessage({ id, type, ...payload } satisfies Request); + } catch (error) { + this.pending.delete(id); + reject(error); + } }); } - async init(name: string, wasmModule: WebAssembly.Module): Promise { - await this.send(INIT, { name, wasmModule }); + async init(name: string, wasmModule: WebAssembly.Module, assetBase: string): Promise { + await this.send(INIT, { name, wasmModule, assetBase }); } async execute(sql: string, params?: unknown[]): Promise<{ rows: Record[] }> { @@ -90,36 +100,55 @@ class WorkerExecutor implements Executor { return { rows }; } - close(): void { - for (const [, handler] of this.pending) { - handler.reject(new Error('Worker terminated')); + async transaction(statements: readonly SqliteStatement[]): Promise { + await this.send(TRANSACTION, { statements }); + } + + async close(): Promise { + if (this.terminated) return; + if (!this.closePromise) { + this.closePromise = this.send(CLOSE).then( + () => { + this.terminated = true; + this.worker.terminate(); + }, + (error) => { + this.terminated = true; + this.worker.terminate(); + throw error; + } + ); } - this.pending.clear(); - this.send(CLOSE).catch(() => {}); - this.worker.terminate(); + await this.closePromise; } } export interface WebSqliteOptions { name: string; + /** Base URL of a compatible wa-sqlite runtime graph, including dist/ and src/. */ + assetBase?: string; worker: Worker | (() => Worker | Promise); } export async function createWebSqlitePersistence(options: WebSqliteOptions): Promise { const { name, worker } = options; + const assetBase = new URL(options.assetBase ?? CDN_BASE, document.baseURI).href.replace( + /\/$/, + '' + ); // Compile WASM in parallel with worker creation for maximum performance. // compileWasmModule() consumes the cache entry and // uses WebAssembly.compileStreaming for streaming compilation. const [resolvedWorker, wasmModule] = await Promise.all([ typeof worker === 'function' ? worker() : worker, - compileWasmModule(), + compileWasmModule(assetBase), ]); const executor = new WorkerExecutor(resolvedWorker); try { - await executor.init(name, wasmModule); + await executor.init(name, wasmModule, assetBase); } catch (error) { resolvedWorker.terminate(); throw new Error(`Failed to initialize: ${error}`); diff --git a/packages/replicate/src/client/persistence/sqlite/worker.ts b/packages/replicate/src/client/persistence/sqlite/worker.ts index 1eaaa23..309e490 100644 --- a/packages/replicate/src/client/persistence/sqlite/worker.ts +++ b/packages/replicate/src/client/persistence/sqlite/worker.ts @@ -1,19 +1,20 @@ -import { initSchema, type Executor } from './schema.js'; - -const CDN_BASE = 'https://wa-sqlite.trestle.inc/v1.0.0'; +import { initSchema, type Executor, type SqliteStatement } from './schema.js'; const INIT = 0; const EXECUTE = 1; const CLOSE = 2; const FLUSH = 3; +const TRANSACTION = 4; interface Request { id: number; - type: typeof INIT | typeof EXECUTE | typeof CLOSE | typeof FLUSH; + type: typeof INIT | typeof EXECUTE | typeof CLOSE | typeof FLUSH | typeof TRANSACTION; name?: string; wasmModule?: WebAssembly.Module; + assetBase?: string; sql?: string; params?: unknown[]; + statements?: readonly SqliteStatement[]; } interface Response { @@ -27,12 +28,18 @@ let sqlite3: any; let db: number; let vfs: any; let mutex: Promise = Promise.resolve(); - -async function init(name: string, wasmModule: WebAssembly.Module): Promise { +let closing = false; +let closePromise: Promise | null = null; + +async function init( + name: string, + wasmModule: WebAssembly.Module, + assetBase: string +): Promise { const [{ default: SQLiteESMFactory }, { IDBBatchAtomicVFS }, SQLite] = await Promise.all([ - import(/* @vite-ignore */ `${CDN_BASE}/dist/wa-sqlite-async.mjs`), - import(/* @vite-ignore */ `${CDN_BASE}/src/examples/IDBBatchAtomicVFS.js`), - import(/* @vite-ignore */ `${CDN_BASE}/src/sqlite-api.js`), + import(/* @vite-ignore */ `${assetBase}/dist/wa-sqlite-async.mjs`), + import(/* @vite-ignore */ `${assetBase}/src/examples/IDBBatchAtomicVFS.js`), + import(/* @vite-ignore */ `${assetBase}/src/sqlite-api.js`), ]); const module = await SQLiteESMFactory({ @@ -52,7 +59,7 @@ async function init(name: string, wasmModule: WebAssembly.Module): Promise db = await sqlite3.open_v2(name); await sqlite3.exec(db, 'PRAGMA cache_size = -8000;'); - await sqlite3.exec(db, 'PRAGMA synchronous = NORMAL;'); + await sqlite3.exec(db, 'PRAGMA synchronous = FULL;'); await sqlite3.exec(db, 'PRAGMA temp_store = MEMORY;'); const executor: Executor = { @@ -63,38 +70,62 @@ async function init(name: string, wasmModule: WebAssembly.Module): Promise sqlite3.close(db); vfs.close(); }, + transaction, }; await initSchema(executor); } -function execute(sql: string, params?: unknown[]): Promise<{ rows: Record[] }> { - const operation = mutex - .catch(() => {}) - .then(async () => { - const rows: Record[] = []; - - for await (const stmt of sqlite3.statements(db, sql)) { - if (params && params.length > 0) { - sqlite3.bind_collection(stmt, params); - } - - const columns: string[] = sqlite3.column_names(stmt); - while ((await sqlite3.step(stmt)) === 100) { - const row = sqlite3.row(stmt); - const obj: Record = {}; - columns.forEach((col: string, i: number) => { - obj[col] = row[i]; - }); - rows.push(obj); - } - } +async function executeNow( + sql: string, + params?: unknown[] +): Promise<{ rows: Record[] }> { + const rows: Record[] = []; + + for await (const stmt of sqlite3.statements(db, sql)) { + if (params && params.length > 0) { + sqlite3.bind_collection(stmt, params); + } + + const columns: string[] = sqlite3.column_names(stmt); + while ((await sqlite3.step(stmt)) === 100) { + const row = sqlite3.row(stmt); + const obj: Record = {}; + columns.forEach((col: string, i: number) => { + obj[col] = row[i]; + }); + rows.push(obj); + } + } + + return { rows }; +} - return { rows }; - }); +function serialize(operation: () => Promise): Promise { + const result = mutex.catch(() => {}).then(operation); + mutex = result; + return result; +} - mutex = operation; - return operation; +function execute(sql: string, params?: unknown[]): Promise<{ rows: Record[] }> { + return serialize(() => executeNow(sql, params)); +} + +function transaction(statements: readonly SqliteStatement[]): Promise { + return serialize(async () => { + await executeNow('BEGIN IMMEDIATE'); + try { + for (const statement of statements) await executeNow(statement.sql, statement.params); + await executeNow('COMMIT'); + } catch (error) { + try { + await executeNow('ROLLBACK'); + } catch { + // Preserve the statement failure that caused the rollback. + } + throw error; + } + }); } self.onmessage = async (e: MessageEvent) => { @@ -103,20 +134,34 @@ self.onmessage = async (e: MessageEvent) => { try { switch (type) { case INIT: - await init(name!, e.data.wasmModule!); + await init(name!, e.data.wasmModule!, e.data.assetBase!); self.postMessage({ id, ok: true } satisfies Response); break; case EXECUTE: { + if (closing) throw new Error('SQLite worker is closing'); const result = await execute(sql!, params); self.postMessage({ id, ok: true, rows: result.rows } satisfies Response); break; } case FLUSH: + if (closing) throw new Error('SQLite worker is closing'); + await mutex; + self.postMessage({ id, ok: true } satisfies Response); + break; + case TRANSACTION: + if (closing) throw new Error('SQLite worker is closing'); + await transaction(e.data.statements!); self.postMessage({ id, ok: true } satisfies Response); break; case CLOSE: - sqlite3.close(db); - vfs.close(); + closing = true; + closePromise ??= mutex + .catch(() => {}) + .then(async () => { + await sqlite3.close(db); + await vfs.close(); + }); + await closePromise; self.postMessage({ id, ok: true } satisfies Response); break; } diff --git a/packages/replicate/src/client/persistence/types.ts b/packages/replicate/src/client/persistence/types.ts index 6ee4f37..66bf535 100644 --- a/packages/replicate/src/client/persistence/types.ts +++ b/packages/replicate/src/client/persistence/types.ts @@ -36,6 +36,10 @@ export interface PersistenceProvider { readonly whenSynced: Promise; destroy(): void; flush?(): Promise; + /** Detach, drain pending writes, and delete this document's durable state. */ + clear?(): Promise; + /** Persist a raw Yjs V2 update, including unresolved dependencies, before applying it. */ + storeUpdate?(update: Uint8Array): Promise; } /** @@ -59,6 +63,8 @@ export interface Persistence { readonly kv: KeyValueStore; /** Direct SQL access for migrations (only available with SQLite persistence) */ readonly db?: MigrationDatabase; + /** Drain providers and release the database/worker. */ + close?(): Promise; } export interface KeyValueStore { diff --git a/packages/replicate/src/client/persistence/writes.ts b/packages/replicate/src/client/persistence/writes.ts new file mode 100644 index 0000000..961889a --- /dev/null +++ b/packages/replicate/src/client/persistence/writes.ts @@ -0,0 +1,28 @@ +/** Serializes durable work, retaining a failed operation for an explicit flush retry. */ +export class PersistenceWrites { + private pending: (() => Promise)[] = []; + private running: Promise | undefined; + + add(write: () => Promise): void { + this.pending.push(write); + void this.flush().catch(() => {}); + } + + flush(): Promise { + if (!this.running) { + this.running = Promise.resolve().then(async () => { + try { + while (this.pending.length) { + await this.pending[0](); + this.pending.shift(); + } + } finally { + // Clear before resolving the drain, so a new flush cannot reuse + // a completed promise while newly queued writes remain pending. + this.running = undefined; + } + }); + } + return this.running; + } +} diff --git a/packages/replicate/src/client/prose.ts b/packages/replicate/src/client/prose.ts index 819b95f..a45c067 100644 --- a/packages/replicate/src/client/prose.ts +++ b/packages/replicate/src/client/prose.ts @@ -92,7 +92,7 @@ export function observeFragment(config: ProseObserverConfig): () => void { logger.debug('Fragment observer registered', { collection, document, field }); const observerHandler = (_events: Y.YEvent[], transaction: Y.Transaction) => { - if (transaction.origin === SERVER_ORIGIN) { + if (transaction.origin === SERVER_ORIGIN || transaction.origin === 'replicate-persisted') { return; } diff --git a/packages/replicate/src/client/services/connection.ts b/packages/replicate/src/client/services/connection.ts new file mode 100644 index 0000000..e8df920 --- /dev/null +++ b/packages/replicate/src/client/services/connection.ts @@ -0,0 +1,111 @@ +import type { ConvexClient } from 'convex/browser'; +import type { Value } from 'convex/values'; + +export interface ConnectionOptions { + /** Start synchronization only after connect() when set to manual. */ + networkMode?: 'auto' | 'manual'; + /** Reject local document mutations and mutable CRDT bindings. */ + readOnly?: boolean; + /** Captured at connect time; disconnect before changing account context. */ + requestArgs?: () => Record; +} + +const protocolKeys = new Set([ + 'seq', + 'limit', + 'document', + 'vector', + 'bytes', + 'material', + 'type', + 'client', + 'action', + 'connected', + 'cursor', + 'profile', + 'user', + 'paginationOpts', + 'threshold', + 'interval', + 'timeout', + 'retain', + 'exclude', + 'collection', +]); + +export function createConnection(client: ConvexClient, options: ConnectionOptions) { + let generation = 0; + let active = false; + let args: Record = {}; + const subscriptions = new Set<() => void>(); + const stale = () => new Error('Synchronization disconnected or superseded'); + const check = (token: number) => { + if (!active || token !== generation) throw stale(); + }; + const wrapped = new Proxy(client, { + get(target, property) { + if (property === 'query' || property === 'mutation') { + return async (reference: any, protocol: any) => { + const token = generation; + check(token); + const result = + property === 'query' + ? await target.query(reference, { ...args, ...protocol }) + : await target.mutation(reference, { ...args, ...protocol }); + check(token); + return result; + }; + } + if (property === 'onUpdate') { + return (reference: any, protocol: any, callback: any, onError?: any) => { + const token = generation; + if (!active) return () => {}; + let listening = true; + const stop = target.onUpdate( + reference, + { ...args, ...protocol }, + (value: any) => { + if (listening && active && token === generation) callback(value); + }, + (error: Error) => { + if (listening && active && token === generation) onError?.(error); + } + ); + const unsubscribe = () => { + if (!listening) return; + listening = false; + subscriptions.delete(unsubscribe); + stop(); + }; + subscriptions.add(unsubscribe); + return unsubscribe; + }; + } + return Reflect.get(target, property, target); + }, + }); + return { + client: wrapped, + get active() { + return active; + }, + get generation() { + return generation; + }, + connect() { + if (active) return; + const supplied = options.requestArgs?.() ?? {}; + for (const key of Object.keys(supplied)) { + if (protocolKeys.has(key)) + throw new Error(`requestArgs cannot override protocol argument: ${key}`); + } + args = structuredClone(supplied); + active = true; + }, + disconnect() { + active = false; + generation++; + for (const stop of subscriptions) stop(); + }, + }; +} diff --git a/packages/replicate/src/client/services/presence.ts b/packages/replicate/src/client/services/presence.ts index 699c22d..48dce0a 100644 --- a/packages/replicate/src/client/services/presence.ts +++ b/packages/replicate/src/client/services/presence.ts @@ -73,6 +73,8 @@ export interface PresenceConfig { heartbeatMs?: number; throttleMs?: number; syncReady?: Promise; + /** Collection integrations supply only vectors covered by durable checkpoints. */ + durableVector?: () => ArrayBuffer | undefined; user?: () => UserIdentity | undefined; anonymousPresence?: AnonymousPresenceConfig; } @@ -161,6 +163,7 @@ export function createPresence(config: PresenceConfig): PresenceProvider { const subscribers = new Set<(state: PresenceState) => void>(); const getVector = (): ArrayBuffer | undefined => { + if (config.durableVector) return config.durableVector(); return Y.encodeStateVector(ydoc).buffer as ArrayBuffer; }; @@ -329,11 +332,13 @@ export function createPresence(config: PresenceConfig): PresenceProvider { } const payload = buildJoinPayload(cursorOverride); - sendWithSingleFlight(payload).then(() => { - if (state === 'joining') { - transitionTo('active'); - } - }); + sendWithSingleFlight(payload) + .then(() => { + if (state === 'joining') { + transitionTo('active'); + } + }) + .catch(() => {}); }; const leavePresence = (): void => { @@ -342,11 +347,13 @@ export function createPresence(config: PresenceConfig): PresenceProvider { transitionTo('leaving'); - sendWithSingleFlight({ action: 'leave' }).then(() => { - if (state === 'leaving') { - transitionTo('idle'); - } - }); + sendWithSingleFlight({ action: 'leave' }) + .then(() => { + if (state === 'leaving') { + transitionTo('idle'); + } + }) + .catch(() => {}); }; const throttledJoin = (): void => { @@ -472,11 +479,13 @@ export function createPresence(config: PresenceConfig): PresenceProvider { if (e.persisted) return; if (state === 'destroyed') return; - convexClient.mutation(api.presence, { - document, - client, - action: 'leave' as const, - }); + convexClient + .mutation(api.presence, { + document, + client, + action: 'leave' as const, + }) + .catch(() => {}); }; globalThis.window.addEventListener('pagehide', handler); @@ -574,11 +583,13 @@ export function createPresence(config: PresenceConfig): PresenceProvider { remoteClientIds.clear(); awareness.emit('update', [{ added: [], updated: [], removed: [] }, 'remote']); - convexClient.mutation(api.presence, { - document, - client, - action: 'leave' as const, - }); + convexClient + .mutation(api.presence, { + document, + client, + action: 'leave' as const, + }) + .catch(() => {}); awareness.destroy(); }, diff --git a/packages/replicate/src/component/_generated/component.ts b/packages/replicate/src/component/_generated/component.ts index 74c6f8c..38a70a1 100644 --- a/packages/replicate/src/component/_generated/component.ts +++ b/packages/replicate/src/component/_generated/component.ts @@ -191,11 +191,16 @@ export type ComponentApi = null, Name >; + migrateSnapshots: FunctionReference< + "mutation", "internal", + { collection: string; limit?: number }, + { migrated: number; done: boolean }, Name + >; recovery: FunctionReference< "query", "internal", { collection: string; document: string; vector: ArrayBuffer }, - { diff?: ArrayBuffer; vector: ArrayBuffer }, + { diff?: ArrayBuffer; vector: ArrayBuffer; known?: boolean }, Name >; runCompaction: FunctionReference< diff --git a/packages/replicate/src/component/mutations.ts b/packages/replicate/src/component/mutations.ts index 139326e..c6377ec 100644 --- a/packages/replicate/src/component/mutations.ts +++ b/packages/replicate/src/component/mutations.ts @@ -39,26 +39,21 @@ async function getNextSeq(ctx: MutationCtx, collection: string): Promise .withIndex('by_collection', (q) => q.eq('collection', collection)) .unique(); - if (existing) { - const nextSeq = existing.seq + 1; - await ctx.db.patch(existing._id, { seq: nextSeq }); - return nextSeq; - } - - // Seed from existing deltas to handle migration from old getNextSeq. - // The previous implementation queried the deltas table directly for max seq. - // Without this seed, the counter would start at 1, creating duplicate seq - // numbers and breaking streaming (clients with cursor > 1 would never - // receive new changes because the stream query uses gt('seq', cursor)). - const latest = await ctx.db + // Include snapshots even when a legacy counter exists below their boundary. + const latestDelta = await ctx.db .query('deltas') .withIndex('by_seq', (q) => q.eq('collection', collection)) .order('desc') .first(); - const startSeq = (latest?.seq ?? 0) + 1; - - await ctx.db.insert('sequences', { collection, seq: startSeq }); - return startSeq; + const latestSnapshot = await ctx.db + .query('snapshots') + .withIndex('by_seq', (q) => q.eq('collection', collection)) + .order('desc') + .first(); + const seq = Math.max(existing?.seq ?? 0, latestDelta?.seq ?? 0, latestSnapshot?.seq ?? 0) + 1; + if (existing) await ctx.db.patch(existing._id, { seq }); + else await ctx.db.insert('sequences', { collection, seq }); + return seq; } // O(1) delta count increment - called when inserting a delta @@ -241,15 +236,6 @@ export const compact = mutation({ const logger = getLogger(['compaction']); const now = Date.now(); - // Get the current max seq at the START of compaction. - // This establishes our snapshot boundary - prevents TOCTOU race conditions - // where new deltas arrive during compaction and get incorrectly included. - const sequenceRecord = await ctx.db - .query('sequences') - .withIndex('by_collection', (q) => q.eq('collection', args.collection)) - .unique(); - const snapshotBoundarySeq = sequenceRecord?.seq ?? 0; - // Query all deltas for this document const allDeltas = await ctx.db .query('deltas') @@ -258,8 +244,8 @@ export const compact = mutation({ ) .collect(); - // Filter to only include deltas within our boundary - const deltas = allDeltas.filter((d) => d.seq <= snapshotBoundarySeq); + // Convex mutations read a consistent transactional snapshot. + const deltas = allDeltas; if (deltas.length === 0) { return { success: true, removed: 0, retained: 0, size: 0 }; @@ -312,15 +298,15 @@ export const compact = mutation({ } } - // Use the boundary seq for the snapshot - this ensures consistency - // with the deltas we included (all have seq <= snapshotBoundarySeq) - const seq = snapshotBoundarySeq; + // Publish a fresh position atomically with replacement and delta removal. + const seq = await getNextSeq(ctx, args.collection); if (existing) { await ctx.db.patch(existing._id, { bytes: merged.buffer as ArrayBuffer, vector: vector.buffer as ArrayBuffer, seq, + publication: 1, created: now, }); } else { @@ -330,6 +316,7 @@ export const compact = mutation({ bytes: merged.buffer as ArrayBuffer, vector: vector.buffer as ArrayBuffer, seq, + publication: 1, created: now, }); } @@ -478,14 +465,6 @@ export const runCompaction = mutation({ const retain = args.retain ?? 0; try { - // Get the current max seq at the START of compaction. - // This establishes our snapshot boundary - prevents TOCTOU race conditions. - const sequenceRecord = await ctx.db - .query('sequences') - .withIndex('by_collection', (q) => q.eq('collection', job.collection)) - .unique(); - const snapshotBoundarySeq = sequenceRecord?.seq ?? 0; - const allDeltas = await ctx.db .query('deltas') .withIndex('by_document', (q) => @@ -493,8 +472,8 @@ export const runCompaction = mutation({ ) .collect(); - // Filter to only include deltas within our boundary - const deltas = allDeltas.filter((d) => d.seq <= snapshotBoundarySeq); + // Convex mutations read a consistent transactional snapshot. + const deltas = allDeltas; if (deltas.length === 0) { await ctx.db.patch(args.id, { status: 'done', completed: now }); @@ -550,14 +529,15 @@ export const runCompaction = mutation({ } } - // Use the boundary seq for the snapshot - const seq = snapshotBoundarySeq; + // Publish a fresh position atomically with replacement and delta removal. + const seq = await getNextSeq(ctx, job.collection); if (snapshot) { await ctx.db.patch(snapshot._id, { bytes: merged.buffer as ArrayBuffer, vector: vector.buffer as ArrayBuffer, seq, + publication: 1, created: now, }); } else { @@ -567,6 +547,7 @@ export const runCompaction = mutation({ bytes: merged.buffer as ArrayBuffer, vector: vector.buffer as ArrayBuffer, seq, + publication: 1, created: now, }); } @@ -646,6 +627,31 @@ export const runCompaction = mutation({ }, }); +/** Republish legacy snapshots in bounded, atomic, restartable batches. + * Call from an application internal mutation until done before enabling streaming. + */ +export const migrateSnapshots = mutation({ + args: { collection: v.string(), limit: v.optional(v.number()) }, + returns: v.object({ migrated: v.number(), done: v.boolean() }), + handler: async (ctx, args) => { + const limit = args.limit ?? 50; + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) { + throw new ConvexError('Migration limit must be an integer between 1 and 100'); + } + const legacy = await ctx.db + .query('snapshots') + .withIndex('by_publication', (q) => + q.eq('collection', args.collection).eq('publication', undefined) + ) + .take(limit + 1); + for (const snapshot of legacy.slice(0, limit)) { + const seq = await getNextSeq(ctx, args.collection); + await ctx.db.patch(snapshot._id, { seq, publication: 1 }); + } + return { migrated: Math.min(limit, legacy.length), done: legacy.length <= limit }; + }, +}); + export const stream = query({ args: { collection: v.string(), @@ -656,77 +662,46 @@ export const stream = query({ returns: streamResultValidator, handler: async (ctx, args) => { const limit = args.limit ?? 100; - // threshold arg kept for API compatibility but no longer used - // (compaction check moved to write mutations for O(1) performance) - - const documents = await ctx.db + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1000) { + throw new ConvexError('Stream limit must be an integer between 1 and 1000'); + } + const legacy = await ctx.db + .query('snapshots') + .withIndex('by_publication', (q) => + q.eq('collection', args.collection).eq('publication', undefined) + ) + .first(); + if (legacy) + throw new ConvexError('Snapshot migration required: run migrateSnapshots until done'); + const deltas = await ctx.db .query('deltas') .withIndex('by_seq', (q) => q.eq('collection', args.collection).gt('seq', args.seq)) .order('asc') - .take(limit); - - if (documents.length > 0) { - const changes = documents.map((doc) => ({ - document: doc.document, - bytes: doc.bytes, - seq: doc.seq, - type: OperationType.Delta, - })); - - const newSeq = documents[documents.length - 1]?.seq ?? args.seq; - - // Compaction eligibility is now checked only during write mutations - // (insertDocument, updateDocument, deleteDocument) via scheduleCompactionIfNeeded. - // This removes the O(n) full collection scan that was running on every subscription update. - - return { - changes, - seq: newSeq, - more: documents.length === limit, - compact: undefined, - }; - } - - const oldest = await ctx.db - .query('deltas') - .withIndex('by_seq', (q) => q.eq('collection', args.collection)) + .take(limit + 1); + const snapshots = await ctx.db + .query('snapshots') + .withIndex('by_seq', (q) => q.eq('collection', args.collection).gt('seq', args.seq)) .order('asc') - .first(); - - if (oldest && args.seq < oldest.seq) { - const snapshots = await ctx.db - .query('snapshots') - .withIndex('by_document', (q) => q.eq('collection', args.collection)) - .collect(); - - if (snapshots.length === 0) { - throw new ConvexError( - `Disparity detected but no snapshots available for collection: ${args.collection}. ` + - `Client seq: ${args.seq}, Oldest delta seq: ${oldest.seq}` - ); - } - - const changes = snapshots.map((s) => ({ - document: s.document, - bytes: s.bytes, - seq: s.seq, + .take(limit + 1); + const candidates = [ + ...deltas.map((d) => ({ + document: d.document, + bytes: d.bytes, + seq: d.seq, + type: OperationType.Delta, + })), + ...snapshots.map((d) => ({ + document: d.document, + bytes: d.bytes, + seq: d.seq, type: OperationType.Snapshot, - })); - - const latestSeq = Math.max(...snapshots.map((s) => s.seq)); - - return { - changes, - seq: latestSeq, - more: false, - compact: undefined, - }; - } - + })), + ].sort((a, b) => a.seq - b.seq); + const changes = candidates.slice(0, limit); return { - changes: [], - seq: args.seq, - more: false, + changes, + seq: changes.at(-1)?.seq ?? args.seq, + more: candidates.length > limit, compact: undefined, }; }, @@ -760,6 +735,7 @@ export const recovery = query({ emptyDoc.destroy(); return { vector: emptyVector.buffer as ArrayBuffer, + known: false, }; } @@ -781,6 +757,7 @@ export const recovery = query({ return { diff: diff.byteLength > 0 ? (diff.buffer as ArrayBuffer) : undefined, vector: serverVector.buffer as ArrayBuffer, + known: true, }; }, }); diff --git a/packages/replicate/src/component/schema.ts b/packages/replicate/src/component/schema.ts index b7ef499..56ebbcb 100644 --- a/packages/replicate/src/component/schema.ts +++ b/packages/replicate/src/component/schema.ts @@ -67,8 +67,12 @@ export default defineSchema({ bytes: v.bytes(), vector: v.bytes(), seq: v.number(), + publication: v.optional(v.literal(1)), created: v.number(), - }).index('by_document', ['collection', 'document']), + }) + .index('by_document', ['collection', 'document']) + .index('by_seq', ['collection', 'seq']) + .index('by_publication', ['collection', 'publication']), sessions: defineTable({ collection: v.string(), diff --git a/packages/replicate/src/server/index.ts b/packages/replicate/src/server/index.ts index 65a98e8..83d08fd 100644 --- a/packages/replicate/src/server/index.ts +++ b/packages/replicate/src/server/index.ts @@ -289,23 +289,34 @@ export class Replicate { // Insert operation if (type === 'insert') { - await ctx.db.insert(collection, { - id: document, - ...(doc as object), - timestamp: Date.now(), - }); + // A disconnected client may retry after the original insert committed. + // The index read and insert share this transaction, so concurrent retries + // cannot create duplicate logical IDs or repeat onInsert side effects. + const existing = await ctx.db + .query(collection) + .withIndex('by_doc_id', (q) => q.eq('id', document)) + .first(); + // Retry payloads can contain newer offline edits. Existing records use + // the normal update path below, including its hook and CRDT merge. + if (!existing) { + await ctx.db.insert(collection, { + id: document, + ...(doc as object), + timestamp: Date.now(), + }); - const result = await ctx.runMutation(component.mutations.insertDocument, { - collection, - document, - bytes, - threshold, - timeout, - retain, - }); + const result = await ctx.runMutation(component.mutations.insertDocument, { + collection, + document, + bytes, + threshold, + timeout, + retain, + }); - await safeHook('onInsert', opts?.onInsert, ctx, doc); - return { success: true, seq: result.seq }; + await safeHook('onInsert', opts?.onInsert, ctx, doc); + return { success: true, seq: result.seq }; + } } // Update operation (fallthrough) @@ -456,12 +467,31 @@ export class Replicate { } if (args.vector !== undefined && args.document !== undefined) { + const [state] = await enrichChangesWithExistence( + ctx, + collection, + [ + { + document: args.document, + bytes: new ArrayBuffer(0), + seq: 0, + type: 'snapshot', + }, + ], + opts?.view + ); + const recoveryResult = await ctx.runQuery(component.mutations.recovery, { collection, document: args.document, vector: args.vector, }); - return { mode: 'recovery' as const, ...recoveryResult }; + return { + mode: 'recovery' as const, + ...recoveryResult, + exists: state.exists, + diff: state.exists ? recoveryResult.diff : undefined, + }; } const result = await ctx.runQuery(component.mutations.stream, { diff --git a/packages/replicate/src/shared/index.ts b/packages/replicate/src/shared/index.ts index 3c2f35c..fcd71fa 100644 --- a/packages/replicate/src/shared/index.ts +++ b/packages/replicate/src/shared/index.ts @@ -245,6 +245,7 @@ export const compactResultValidator = v.object({ * Recovery query result with optional diff and state vector. */ export const recoveryResultValidator = v.object({ + known: v.optional(v.boolean()), diff: v.optional(v.bytes()), vector: v.bytes(), }); diff --git a/packages/replicate/src/test/assets.test.ts b/packages/replicate/src/test/assets.test.ts new file mode 100644 index 0000000..390939a --- /dev/null +++ b/packages/replicate/src/test/assets.test.ts @@ -0,0 +1,57 @@ +// @vitest-environment node +import { test, expect, vi } from 'vitest'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { mkdtemp, readFile, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createHash } from 'node:crypto'; +import { compileWasmModule } from '../client/persistence/sqlite/preload'; + +test('compileWasmModule rejects in Node without fetching browser assets', async () => { + const fetch = vi.fn(() => Promise.reject(new Error('fetch should not run'))); + vi.stubGlobal('fetch', fetch); + try { + await expect(compileWasmModule('/local-runtime')).rejects.toThrow( + 'compileWasmModule is browser-only' + ); + expect(fetch).not.toHaveBeenCalled(); + } finally { + vi.unstubAllGlobals(); + } +}); + +test('asset CLI installs into a new directory and refreshes assets on repeated builds', async () => { + const directory = await mkdtemp(join(tmpdir(), 'replicate-assets-')); + const destination = join(directory, 'public', 'sqlite'); + const script = fileURLToPath(new URL('../../scripts/copy-sqlite-assets.mjs', import.meta.url)); + const run = () => promisify(execFile)(process.execPath, [script, destination]); + try { + await run(); + const manifest = JSON.parse(await readFile(join(destination, 'manifest.json'), 'utf8')); + expect(manifest).toMatchObject({ + package: 'wa-sqlite', + version: '1.1.2', + revision: '07ad48cf2f682d279f9cd69818cc1d39b2ccda86', + }); + expect(Object.keys(manifest.sha256)).toEqual([ + ...manifest.dependencyFiles, + ...manifest.packageFiles, + ]); + const verify = async () => { + for (const [file, expected] of Object.entries(manifest.sha256)) { + const bytes = await readFile(join(destination, file)); + expect(createHash('sha256').update(bytes).digest('hex'), file).toBe(expected); + } + }; + await verify(); + await writeFile(join(destination, 'dist', 'wa-sqlite-async.wasm'), 'previous build'); + await writeFile(join(destination, 'unrelated.txt'), 'preserve'); + await run(); + await verify(); + expect(await readFile(join(destination, 'unrelated.txt'), 'utf8')).toBe('preserve'); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/packages/replicate/src/test/component.test.ts b/packages/replicate/src/test/component.test.ts index 6974b9f..eac41f4 100644 --- a/packages/replicate/src/test/component.test.ts +++ b/packages/replicate/src/test/component.test.ts @@ -63,6 +63,13 @@ describe('Replicate Component Mutations', () => { }); describe('scheduleCompaction', () => { + const validUpdate = () => { + const doc = new Y.Doc(); + doc.getMap('fields').set('title', 'scheduled'); + const update = Y.encodeStateAsUpdateV2(doc); + doc.destroy(); + return update.buffer as ArrayBuffer; + }; it('should schedule compaction for a document', async () => { vi.useFakeTimers(); @@ -70,7 +77,7 @@ describe('Replicate Component Mutations', () => { await t.mutation(api.mutations.insertDocument, { collection: 'test-collection', document: 'test-doc', - bytes: new Uint8Array([1, 2, 3]).buffer as ArrayBuffer, + bytes: validUpdate(), }); const result = await t.mutation(api.mutations.scheduleCompaction, { @@ -94,7 +101,7 @@ describe('Replicate Component Mutations', () => { await t.mutation(api.mutations.insertDocument, { collection: 'test-collection', document: 'test-doc', - bytes: new Uint8Array([1, 2, 3]).buffer as ArrayBuffer, + bytes: validUpdate(), }); // Schedule first compaction diff --git a/packages/replicate/src/test/discovery.test.ts b/packages/replicate/src/test/discovery.test.ts new file mode 100644 index 0000000..b0bb694 --- /dev/null +++ b/packages/replicate/src/test/discovery.test.ts @@ -0,0 +1,312 @@ +import { afterEach, expect, test, vi } from 'vitest'; +import { convexTest } from 'convex-test'; +import * as Y from 'yjs'; +import { api } from '../component/_generated/api'; +import schema from '../component/schema'; +import { modules } from '../component/test.setup'; + +const collection = 'discovery'; +const make = () => convexTest(schema, modules); +type Backend = ReturnType; +function document(title: string) { + const doc = new Y.Doc(); + doc.getMap('fields').set('title', title); + return doc; +} +const bytes = (doc: Y.Doc) => Y.encodeStateAsUpdateV2(doc).buffer as ArrayBuffer; +async function insert(t: Backend, id: string, doc: Y.Doc) { + await t.mutation(api.mutations.insertDocument, { collection, document: id, bytes: bytes(doc) }); +} +async function compact(t: Backend, id: string) { + await t.mutation(api.mutations.compact, { collection, document: id }); +} +async function read( + t: Backend, + limit: number, + cursor = 0, + between?: (page: number) => Promise +) { + const docs = new Map(); + const sequences: number[] = []; + for (let page = 0; page < 30; page++) { + const result = await t.query(api.mutations.stream, { collection, seq: cursor, limit }); + expect(result.changes.length).toBeLessThanOrEqual(limit); + for (const change of result.changes) { + expect(change.seq).toBeGreaterThan(cursor); + sequences.push(change.seq); + const doc = docs.get(change.document) ?? new Y.Doc(); + docs.set(change.document, doc); + Y.applyUpdateV2(doc, new Uint8Array(change.bytes)); + } + if (!result.changes.length) break; + cursor = result.seq; + await between?.(page); + } + const titles = Object.fromEntries( + [...docs].map(([id, doc]) => [id, doc.getMap('fields').get('title')]) + ); + for (const doc of docs.values()) doc.destroy(); + expect(new Set(sequences).size).toBe(sequences.length); + return { titles, cursor }; +} +afterEach(() => vi.useRealTimers()); + +test.each([1, 2])( + 'fresh client discovers a fully compacted collection with page limit %s', + async (limit) => { + const t = make(); + for (const id of ['a', 'b', 'c']) { + const doc = document(id); + await insert(t, id, doc); + await compact(t, id); + doc.destroy(); + } + expect(await t.run((ctx) => ctx.db.query('deltas').collect())).toEqual([]); + expect((await read(t, limit)).titles).toEqual({ a: 'a', b: 'b', c: 'c' }); + } +); + +test.each([1, 2])( + 'mixed snapshots and incremental deltas converge with page limit %s', + async (limit) => { + const t = make(); + const a = document('base'); + await insert(t, 'a', a); + await compact(t, 'a'); + const vector = Y.encodeStateVector(a); + a.getMap('fields').set('title', 'latest'); + await t.mutation(api.mutations.updateDocument, { + collection, + document: 'a', + bytes: Y.encodeStateAsUpdateV2(a, vector).buffer as ArrayBuffer, + }); + const b = document('b'); + await insert(t, 'b', b); + expect((await read(t, limit)).titles).toEqual({ a: 'latest', b: 'b' }); + a.destroy(); + b.destroy(); + } +); + +test.each([1, 2])( + 'writes and compaction between pages remain discoverable at limit %s', + async (limit) => { + const t = make(); + const a = document('a'); + const b = document('b'); + await insert(t, 'a', a); + await insert(t, 'b', b); + const result = await read(t, limit, 0, async (page) => { + if (page !== 0) return; + a.getMap('fields').set('title', 'new-a'); + await insert(t, 'a', a); + await compact(t, 'a'); + await compact(t, 'b'); + const c = document('c'); + await insert(t, 'c', c); + await compact(t, 'c'); + c.destroy(); + }); + expect(result.titles).toEqual({ a: 'new-a', b: 'b', c: 'c' }); + a.destroy(); + b.destroy(); + } +); + +test.each([undefined, 2])( + 'legacy shared snapshot positions migrate above cursor with counter %s', + async (counter) => { + const t = make(); + await t.run(async (ctx) => { + if (counter !== undefined) await ctx.db.insert('sequences', { collection, seq: counter }); + for (const id of ['a', 'b', 'c']) { + const doc = document(id); + await ctx.db.insert('snapshots', { + collection, + document: id, + bytes: bytes(doc), + vector: Y.encodeStateVector(doc).buffer as ArrayBuffer, + seq: 10, + created: 0, + }); + doc.destroy(); + } + }); + await expect(t.query(api.mutations.stream, { collection, seq: 10, limit: 1 })).rejects.toThrow( + 'migration required' + ); + const first = await t.mutation(api.mutations.migrateSnapshots, { collection, limit: 1 }); + expect(first).toEqual({ migrated: 1, done: false }); + await expect(t.query(api.mutations.stream, { collection, seq: 10, limit: 1 })).rejects.toThrow( + 'migration required' + ); + // A write between restartable migration batches uses the same allocator. + const d = document('d'); + await insert(t, 'd', d); + d.destroy(); + const second = await t.mutation(api.mutations.migrateSnapshots, { collection, limit: 2 }); + expect(second).toEqual({ migrated: 2, done: true }); + expect(await t.mutation(api.mutations.migrateSnapshots, { collection, limit: 1 })).toEqual({ + migrated: 0, + done: true, + }); + expect((await read(t, 1, 10)).titles).toEqual({ a: 'a', b: 'b', c: 'c', d: 'd' }); + } +); + +test('legacy base republished after its incremental delta still supports discovery and recovery', async () => { + const t = make(); + const doc = document('old'); + const base = bytes(doc); + const vector = Y.encodeStateVector(doc); + doc.getMap('fields').set('title', 'new'); + await t.run(async (ctx) => { + await ctx.db.insert('snapshots', { + collection, + document: 'a', + bytes: base, + vector: vector.buffer as ArrayBuffer, + seq: 10, + created: 0, + }); + await ctx.db.insert('deltas', { + collection, + document: 'a', + bytes: Y.encodeStateAsUpdateV2(doc, vector).buffer as ArrayBuffer, + seq: 11, + }); + }); + await t.mutation(api.mutations.migrateSnapshots, { collection }); + expect((await read(t, 1)).titles).toEqual({ a: 'new' }); + const recovered = new Y.Doc(); + const result = await t.query(api.mutations.recovery, { + collection, + document: 'a', + vector: Y.encodeStateVector(recovered).buffer as ArrayBuffer, + }); + Y.applyUpdateV2(recovered, new Uint8Array(result.diff!)); + expect(recovered.getMap('fields').get('title')).toBe('new'); + doc.destroy(); + recovered.destroy(); +}); + +test('scheduled compaction publishes snapshots and preserves configured delta retention', async () => { + vi.useFakeTimers(); + const t = make(); + const doc = document('one'); + await insert(t, 'a', doc); + doc.getMap('fields').set('title', 'two'); + await insert(t, 'a', doc); + const scheduled = await t.mutation(api.mutations.scheduleCompaction, { + collection, + document: 'a', + retain: 1, + }); + expect(scheduled.status).toBe('scheduled'); + await t.finishAllScheduledFunctions(vi.runAllTimers); + const deltas = await t.run((ctx) => ctx.db.query('deltas').collect()); + expect(deltas).toHaveLength(1); + expect((await read(t, 1)).titles).toEqual({ a: 'two' }); + doc.destroy(); +}); + +test('snapshot publication does not acknowledge data for a connected old client', async () => { + const t = make(); + const doc = document('a'); + await insert(t, 'a', doc); + await t.mutation(api.mutations.mark, { + collection, + document: 'a', + client: 'old', + seq: 0, + vector: Y.encodeStateVector(new Y.Doc()).buffer as ArrayBuffer, + }); + await t.run(async (ctx) => { + const session = await ctx.db.query('sessions').first(); + await ctx.db.patch(session!._id, { connected: true }); + }); + await compact(t, 'a'); + expect((await t.run((ctx) => ctx.db.query('deltas').collect())).length).toBe(1); + expect((await read(t, 1)).titles).toEqual({ a: 'a' }); + doc.destroy(); +}); +test('seeded interleavings converge after legacy migration, writes, compaction, and paging', async () => { + for (let seed = 1; seed <= 24; seed++) { + let state = seed; + const nextRandom = (max: number) => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state % max; + }; + const t = convexTest(schema, modules), + collection = 'interleavings'; + const sourceDocuments = Array.from({ length: 7 }, (_, id) => { + const doc = new Y.Doc(); + doc.getMap('fields').set('title', String(id)); + return doc; + }); + const receivedDocuments = sourceDocuments.map(() => new Y.Doc()); + let cursor = 10; + await t.run(async (ctx) => { + for (let id = 0; id < 7; id++) { + const doc = sourceDocuments[id]; + await ctx.db.insert('snapshots', { + collection, + document: String(id), + bytes: Y.encodeStateAsUpdateV2(doc).buffer as ArrayBuffer, + vector: Y.encodeStateVector(doc).buffer as ArrayBuffer, + seq: 10, + created: 0, + }); + } + }); + const write = async (id: number, value: number) => { + const doc = sourceDocuments[id], + vector = Y.encodeStateVector(doc); + doc.getMap('fields').set('title', String(value)); + await t.mutation(api.mutations.updateDocument, { + collection, + document: String(id), + bytes: Y.encodeStateAsUpdateV2(doc, vector).buffer as ArrayBuffer, + }); + }; + for (let n = 0; n < 7; n++) { + await t.mutation(api.mutations.migrateSnapshots, { collection, limit: 1 }); + await write(nextRandom(7), n + 100); + if (nextRandom(2)) + await t.mutation(api.mutations.compact, { collection, document: String(nextRandom(7)) }); + } + const readPage = async () => { + const response = await t.query(api.mutations.stream, { + collection, + seq: cursor, + limit: nextRandom(2) + 1, + }); + for (const change of response.changes) { + expect(change.seq).toBeGreaterThan(cursor); + Y.applyUpdateV2(receivedDocuments[Number(change.document)], new Uint8Array(change.bytes)); + } + cursor = response.seq; + return response.changes.length; + }; + for (let n = 0; n < 80; n++) { + const action = nextRandom(3); + if (action === 0) await write(nextRandom(7), n + 200); + else if (action === 1) + await t.mutation(api.mutations.compact, { collection, document: String(nextRandom(7)) }); + else await readPage(); + } + let drained = false; + for (let remaining = 200; remaining > 0; remaining--) { + if ((await readPage()) === 0) { + drained = true; + break; + } + } + expect(drained, 'stream drains after writes stop').toBe(true); + expect( + receivedDocuments.map((doc) => doc.getMap('fields').toJSON()), + `seed ${seed}` + ).toEqual(sourceDocuments.map((doc) => doc.getMap('fields').toJSON())); + for (const doc of [...sourceDocuments, ...receivedDocuments]) doc.destroy(); + } +}, 30000); diff --git a/packages/replicate/src/test/offline.test.ts b/packages/replicate/src/test/offline.test.ts new file mode 100644 index 0000000..ffe73c6 --- /dev/null +++ b/packages/replicate/src/test/offline.test.ts @@ -0,0 +1,1678 @@ +// @vitest-environment node +import { afterEach, expect, test, vi } from 'vitest'; +import { DatabaseSync } from 'node:sqlite'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import * as Y from 'yjs'; +import { makeFunctionReference, getFunctionName } from 'convex/server'; +import { v } from 'convex/values'; +import { collection } from '../client/collection'; +import { schema } from '../server'; +import { createNativeSqlitePersistence } from '../client/persistence/sqlite/native'; +import { createCustomPersistence } from '../client/persistence/custom'; +import { createWebEncryptionPersistence } from '../client/persistence/encrypted/web'; +import { createConnection } from '../client/services/connection'; +import { fragmentToJSON } from '../client/merge'; +import type { ConnectionOptions } from '../client/services/connection'; +import type { Persistence } from '../client/persistence/types'; + +const api = Object.fromEntries( + ['material', 'delta', 'replicate', 'presence', 'session'].map((name) => [ + name, + makeFunctionReference(`offline:${name}`), + ]) +) as any; +const definition = schema.define({ + shape: v.object({ id: v.string(), title: v.string(), note: v.optional(v.string()) }), + defaults: {}, + history: {}, +}); +const proseDefinition = schema.define({ + shape: v.object({ id: v.string(), title: v.string(), content: schema.prose() }), + defaults: {}, + history: {}, +}); +const disposals: (() => Promise)[] = []; +afterEach(async () => { + for (const dispose of disposals.splice(0).reverse()) await dispose(); +}); +const deferred = () => { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + return { promise, resolve, reject }; +}; +function transport() { + const subscriptions: { args: any; callback: (value: any) => void; stopped: boolean }[] = []; + const client = { + query: vi.fn(async () => ({ mode: 'recovery', exists: true, diff: null }) as any), + mutation: vi.fn(async () => ({ success: true, seq: 1 })), + onUpdate: vi.fn((_ref: any, args: any, callback: (value: any) => void) => { + const sub = { args, callback, stopped: false }; + subscriptions.push(sub); + return () => { + if (sub.stopped) throw new Error('Subscription already stopped'); + sub.stopped = true; + }; + }), + }; + return { + client, + subscriptions, + emit: (value: any) => + subscriptions.filter((s) => !s.stopped && 'seq' in s.args).forEach((s) => s.callback(value)), + }; +} +async function disk(path?: string, before?: (sql: string, params?: unknown[]) => Promise) { + if (!path) { + const directory = await mkdtemp(join(tmpdir(), 'replicate-contract-')); + disposals.push(() => rm(directory, { recursive: true, force: true })); + path = join(directory, 'replica.sqlite'); + } + const db = new DatabaseSync(path); + let closed = false; + const persistence = await createNativeSqlitePersistence( + { + async execute(sql, params = []) { + await before?.(sql, params); + const statement = db.prepare(sql); + if (statement.columns().length) + return { rows: statement.all(...(params as any[])) as Record[] }; + statement.run(...(params as any[])); + return { rows: [] }; + }, + close() { + closed = true; + db.close(); + }, + }, + path + ); + return { persistence, path, database: db, isClosed: () => closed }; +} +async function seed(p: Persistence, id = 'a') { + const doc = new Y.Doc(); + doc.clientID = 1; + const provider = p.createDocPersistence(`offline:${id}`, doc); + await provider.whenSynced; + doc.getMap('fields').set('title', 'persisted'); + await provider.flush!(); + const state = Y.encodeStateAsUpdateV2(doc); + provider.destroy(); + doc.destroy(); + return state; +} +async function open(p: Persistence, network = transport(), options: ConnectionOptions = {}) { + const lazy = collection.create<{ id: string; title: string }>({ + schema: definition, + persistence: async () => p, + config: () => ({ convexClient: network.client as any, api, getKey: (row) => row.id }), + ...options, + }); + await lazy.init(); + const rows = lazy.get(); + await rows.preload(); + disposals.push(async () => { + await rows.disconnect(); + await rows.cleanup(); + }); + return { rows, network }; +} +async function seedProse(p: Persistence) { + const doc = new Y.Doc(); + doc.clientID = 1; + const provider = p.createDocPersistence('offline:a', doc); + await provider.whenSynced; + const fields = doc.getMap('fields'); + fields.set('title', 'base'); + fields.set('content', new Y.XmlFragment()); + await provider.flush!(); + const state = Y.encodeStateAsUpdateV2(doc); + provider.destroy(); + doc.destroy(); + return state; +} +async function openProse(p: Persistence, network = transport()) { + const lazy = collection.create<{ id: string; title: string; content: any }>({ + schema: proseDefinition, + persistence: async () => p, + config: () => ({ convexClient: network.client as any, api, getKey: (row) => row.id }), + networkMode: 'manual', + }); + await lazy.init(); + const rows = lazy.get(); + await rows.preload(); + disposals.push(async () => { + await rows.disconnect(); + await rows.cleanup(); + }); + return { rows, network }; +} +function proseMaterial(doc: Y.Doc) { + const fields = doc.getMap('fields'); + return { + title: fields.get('title'), + content: fragmentToJSON(fields.get('content') as Y.XmlFragment), + id: 'a', + }; +} +function change(id: string, seq: number, title = 'remote') { + const doc = new Y.Doc(); + doc.clientID = 2; + doc.getMap('fields').set('title', title); + const bytes = Y.encodeStateAsUpdateV2(doc).buffer; + doc.destroy(); + return { document: id, seq, bytes, type: 'delta', exists: true }; +} +const page = (...changes: any[]) => ({ + mode: 'stream', + changes, + seq: changes.at(-1).seq, + more: false, +}); +function proseTitleChange(base: Uint8Array, title: string, seq: number) { + const doc = new Y.Doc(); + doc.clientID = 2; + Y.applyUpdateV2(doc, base); + const vector = Y.encodeStateVector(doc); + doc.getMap('fields').set('title', title); + const bytes = Y.encodeStateAsUpdateV2(doc, vector).buffer; + doc.destroy(); + return page({ document: 'a', seq, bytes, type: 'delta', exists: true }); +} + +test.each(['disconnect', 'reset'] as const)( + '%s followed by reconnect and cleanup stops each transport subscription once', + async (action) => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + const { rows, network } = await open(db.persistence, transport(), { networkMode: 'manual' }); + await rows.connect(); + const first = [...network.subscriptions]; + expect(first.length).toBeGreaterThan(0); + await rows[action](); + expect(first.every((sub) => sub.stopped)).toBe(true); + await rows.connect(); + expect(network.subscriptions.some((sub) => !sub.stopped)).toBe(true); + await rows.disconnect(); + await rows.cleanup(); + expect(network.subscriptions.every((sub) => sub.stopped)).toBe(true); + } +); + +test('unsubscribing suppresses queued callbacks without disconnecting other subscriptions', () => { + const callbacks: { value: (value: unknown) => void; error: (error: Error) => void }[] = []; + const stop = vi.fn(); + const connection = createConnection( + { + onUpdate: (_ref: unknown, _args: unknown, value: any, error: any) => { + callbacks.push({ value, error }); + return stop; + }, + } as any, + {} + ); + connection.connect(); + const value = vi.fn(); + const error = vi.fn(); + const remaining = vi.fn(); + const unsubscribe = connection.client.onUpdate(api.delta, {}, value, error); + connection.client.onUpdate(api.delta, {}, remaining); + unsubscribe(); + unsubscribe(); + callbacks[0].value({}); + callbacks[0].error(new Error('queued')); + callbacks[1].value({}); + expect(value).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + expect(remaining).toHaveBeenCalledOnce(); + expect(stop).toHaveBeenCalledOnce(); + connection.disconnect(); + expect(stop).toHaveBeenCalledTimes(2); +}); + +test('manual session subscribers start on connect and reject old callbacks after reconnect', async () => { + const first = await disk(); + disposals.push(() => first.persistence.close!()); + const net = transport(); + const { rows } = await open(first.persistence, net, { networkMode: 'manual' }); + const listener = vi.fn(); + const stop = rows.session.subscribe(listener); + expect(net.client.onUpdate).not.toHaveBeenCalled(); + await rows.connect(); + const original = net.subscriptions.find((sub) => sub.args.connected === true)!; + expect(original).toBeDefined(); + listener.mockClear(); + original.callback([]); + expect(listener).toHaveBeenCalledOnce(); + await rows.disconnect(); + await rows.connect(); + listener.mockClear(); + original.callback([]); + expect(listener).not.toHaveBeenCalled(); + const current = net.subscriptions.find((sub) => sub.args.connected === true && !sub.stopped)!; + expect(current).not.toBe(original); + current.callback([]); + expect(listener).toHaveBeenCalledOnce(); + stop(); +}); + +test.each(['insert', 'recovery'])( + 'reset drains accepted %s pending-marker deletion', + async (source) => { + const first = await disk(); + disposals.push(() => first.persistence.close!()); + if (source === 'recovery') { + await seed(first.persistence); + await first.persistence.kv.set('pending-create:offline:a', true); + } + const entered = deferred(); + const release = deferred(); + const del = first.persistence.kv.del.bind(first.persistence.kv); + let intercept = true; + first.persistence.kv.del = async (key) => { + if (intercept && key === 'pending-create:offline:a') { + intercept = false; + entered.resolve(); + await release.promise; + } + await del(key); + }; + const { rows } = await open(first.persistence); + if (source === 'insert') { + void rows.insert({ id: 'a', title: 'old' }).isPersisted.promise.catch(() => {}); + } + await entered.promise; + let resetDone = false; + const reset = rows.reset().then(() => { + resetDone = true; + }); + await new Promise((resolve) => setTimeout(resolve, 30)); + const completedBeforeRelease = resetDone; + release.resolve(); + await reset; + expect(completedBeforeRelease).toBe(false); + expect(await first.persistence.kv.get('pending-create:offline:a')).toBeUndefined(); + } +); + +test('recovery and single-document stream changes preserve untouched row identities', async () => { + const first = await disk(); + disposals.push(() => first.persistence.close!()); + await seed(first.persistence, 'a'); + await seed(first.persistence, 'b'); + const { rows, network } = await open(first.persistence, transport(), { networkMode: 'manual' }); + const originalA = rows.get('a'); + const originalB = rows.get('b'); + await rows.connect(); + await vi.waitFor(() => expect(network.client.query).toHaveBeenCalledTimes(2)); + await rows.disconnect(); + expect(rows.get('a')).toBe(originalA); + expect(rows.get('b')).toBe(originalB); + await rows.connect(); + network.emit(page(change('a', 1, 'changed'))); + await vi.waitFor(() => expect(rows.get('a')?.title).toBe('changed')); + expect(rows.get('b')).toBe(originalB); +}); + +test('targeted stream publication removes fields missing from the current document', async () => { + const first = await disk(); + disposals.push(() => first.persistence.close!()); + const { rows, network } = await open(first.persistence, transport(), { networkMode: 'manual' }); + await rows.connect(); + const remote = new Y.Doc(); + remote.getMap('fields').set('title', 'record'); + remote.getMap('fields').set('note', 'removed later'); + network.emit(page({ ...change('a', 1), bytes: Y.encodeStateAsUpdateV2(remote).buffer })); + await vi.waitFor(() => expect(rows.get('a')).toHaveProperty('note', 'removed later')); + const vector = Y.encodeStateVector(remote); + remote.getMap('fields').delete('note'); + network.emit(page({ ...change('a', 2), bytes: Y.encodeStateAsUpdateV2(remote, vector).buffer })); + await vi.waitFor(() => expect(rows.get('a')).not.toHaveProperty('note')); + expect(rows.get('a')?.title).toBe('record'); + remote.destroy(); +}); + +test('recovery and streaming flush only the documents whose data changed', async () => { + const first = await disk(); + disposals.push(() => first.persistence.close!()); + const count = 20; + for (let id = 0; id < count; id++) await seed(first.persistence, String(id)); + let flushes = 0; + const create = first.persistence.createDocPersistence.bind(first.persistence); + first.persistence.createDocPersistence = (key, doc) => { + const provider = create(key, doc); + const flush = provider.flush!.bind(provider); + provider.flush = async () => { + flushes++; + await flush(); + }; + return provider; + }; + const network = transport(); + network.client.query.mockResolvedValue({ + mode: 'recovery', + exists: true, + diff: change('unused', 1).bytes, + }); + const { rows } = await open(first.persistence, network, { + networkMode: 'manual', + readOnly: true, + }); + await rows.connect(); + await vi.waitFor(() => + expect([...rows.values()].filter((row) => row.title === 'remote')).toHaveLength(count) + ); + expect(flushes).toBeLessThanOrEqual(count * 2); + flushes = 0; + network.emit(page(change('new', 1))); + await vi.waitFor(() => expect(rows.get('new')?.title).toBe('remote')); + expect(flushes).toBeLessThanOrEqual(2); +}); + +test.each(['data', 'cursor'])( + 'disconnect during %s persistence keeps durable rows readable on offline reconnect', + async (phase) => { + const entered = deferred(); + const release = deferred(); + const first = await disk(undefined, async (sql, params) => { + const key = phase === 'data' ? 'offline:a' : 'cursor:offline'; + if (sql.startsWith('INSERT') && params?.[0] === key) { + entered.resolve(); + await release.promise; + } + }); + disposals.push(() => first.persistence.close!()); + const network = transport(); + network.client.query.mockImplementation(() => new Promise(() => {})); + const { rows } = await open(first.persistence, network, { + networkMode: 'manual', + readOnly: true, + }); + await rows.connect(); + network.emit(page(change('a', 1))); + await entered.promise; + const disconnect = rows.disconnect(); + release.resolve(); + await disconnect; + const expectedCursor = phase === 'cursor' ? 1 : 0; + expect((await first.persistence.kv.get('cursor:offline')) ?? 0).toBe(expectedCursor); + await rows.connect(); + expect(network.subscriptions.at(-1)?.args.seq).toBe(expectedCursor); + expect(rows.get('a')?.title).toBe('remote'); + } +); + +test('disk rows become ready while all backend promises remain unresolved', async () => { + const first = await disk(); + await seed(first.persistence); + await first.persistence.close!(); + const reopened = await disk(first.path); + disposals.push(() => reopened.persistence.close!()); + const net = transport(); + net.client.query.mockImplementation(() => new Promise(() => {})); + net.client.mutation.mockImplementation(() => new Promise(() => {})); + const { rows } = await open(reopened.persistence, net); + expect(rows.get('a')?.title).toBe('persisted'); + await rows.disconnect(); +}); + +test.each(['stream', 'recovery'])( + '%s absence survives actual SQLite close and reopen', + async (source) => { + const first = await disk(); + await seed(first.persistence); + const net = transport(); + if (source === 'recovery') + net.client.query.mockResolvedValue({ mode: 'recovery', exists: false }); + const { rows } = await open(first.persistence, net); + if (source === 'stream') { + await vi.waitFor(() => expect(net.subscriptions.length).toBeGreaterThan(0)); + net.emit(page({ ...change('a', 1), exists: false })); + } + await vi.waitFor(() => expect(rows.get('a')).toBeUndefined()); + await rows.disconnect(); + await rows.cleanup(); + disposals.pop(); + await first.persistence.close!(); + const reopened = await disk(first.path); + disposals.push(() => reopened.persistence.close!()); + const next = await open(reopened.persistence, transport(), { networkMode: 'manual' }); + expect(next.rows.get('a')).toBeUndefined(); + expect(await reopened.persistence.listDocuments('offline')).toEqual([]); + } +); + +test('manual read-only replica performs no transport before connect and never writes documents', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + await seed(db.persistence); + const net = transport(); + const { rows } = await open(db.persistence, net, { + networkMode: 'manual', + readOnly: true, + requestArgs: () => ({ guest: 'credential' }), + }); + rows.session.subscribe(() => {}); + expect(net.client.query).not.toHaveBeenCalled(); + expect(net.client.onUpdate).not.toHaveBeenCalled(); + expect(net.client.mutation).not.toHaveBeenCalled(); + await rows.connect(); + net.emit(page(change('a', 1, 'server owned'))); + await vi.waitFor(() => expect(rows.get('a')?.title).toBe('server owned')); + const transaction = rows.update('a', (row) => { + row.title = 'forbidden'; + }); + await expect(transaction.isPersisted.promise).rejects.toThrow('read-only'); + expect( + net.client.mutation.mock.calls.every( + ([ref]: any) => getFunctionName(ref) !== 'offline:replicate' + ) + ).toBe(true); + for (const calls of [ + net.client.query.mock.calls, + net.client.mutation.mock.calls, + net.client.onUpdate.mock.calls, + ]) { + for (const call of calls as any[]) expect(call[1].guest).toBe('credential'); + } +}); + +test('writable default still inserts and sends document updates', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + const { rows, network } = await open(db.persistence); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + let seq = 0; + network.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate') + network.emit(page({ ...args, seq: ++seq, exists: true })); + return { success: true, seq }; + }); + await rows.insert({ id: 'new', title: 'write' }).isPersisted.promise; + await vi.waitFor(() => expect(rows.get('new')).toBeDefined()); + await rows.update('new', (row) => { + row.title = 'edited'; + }).isPersisted.promise; + expect( + network.client.mutation.mock.calls.filter( + ([ref]: any) => getFunctionName(ref) === 'offline:replicate' + ) + ).toHaveLength(2); + await vi.waitFor(() => expect(rows.get('new')?.title).toBe('edited')); +}); + +test.each(['insert', 'update', 'delete'] as const)( + 'local %s waits for durable persistence before remote acknowledgement', + async (action) => { + const entered = deferred(); + const release = deferred(); + const document = action === 'insert' ? 'new' : 'a'; + let block = false; + const db = await disk(undefined, async (sql, params) => { + if ( + block && + sql.startsWith('INSERT INTO deltas') && + params?.[0] === `offline:${document}` + ) { + entered.resolve(); + await release.promise; + } + }); + disposals.push(() => db.persistence.close!()); + if (action !== 'insert') await seed(db.persistence); + const { rows, network } = await open(db.persistence, transport(), { + networkMode: 'manual', + }); + await rows.connect(); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + block = true; + const transaction = + action === 'insert' + ? rows.insert({ id: document, title: 'local' }) + : action === 'update' + ? rows.update(document, (row) => { + row.title = 'local'; + }) + : rows.delete(document); + let outcome = 'pending'; + const persisted = transaction.isPersisted.promise.then( + () => { + outcome = 'resolved'; + }, + (error) => { + outcome = 'rejected'; + throw error; + } + ); + await entered.promise; + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(outcome).toBe('pending'); + expect( + network.client.mutation.mock.calls.some( + ([ref]: any) => getFunctionName(ref) === 'offline:replicate' + ) + ).toBe(false); + if (action === 'insert') { + expect( + db.database + .prepare("SELECT value FROM kv WHERE key = 'pending-create:offline:new'") + .get() + ).toBeDefined(); + } + release.resolve(); + await persisted; + expect(outcome).toBe('resolved'); + expect( + network.client.mutation.mock.calls.some( + ([ref, args]: any) => + getFunctionName(ref) === 'offline:replicate' && args.type === action + ) + ).toBe(true); + if (action === 'insert') { + expect( + db.database + .prepare("SELECT value FROM kv WHERE key = 'pending-create:offline:new'") + .get() + ).toBeUndefined(); + } + disposals.pop(); + disposals.pop(); + db.database.close(); + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const next = await open(reopened.persistence, transport(), { networkMode: 'manual' }); + if (action === 'delete') expect(next.rows.get(document)).toBeUndefined(); + else expect(next.rows.get(document)?.title).toBe('local'); + } +); + +test.each(['insert', 'update', 'delete'] as const)( + 'failed local %s persistence rejects before remote acknowledgement', + async (action) => { + const document = action === 'insert' ? 'new' : 'a'; + let fail = false; + const db = await disk(undefined, async (sql, params) => { + if ( + fail && + sql.startsWith('INSERT INTO deltas') && + params?.[0] === `offline:${document}` + ) { + throw new Error('disk unavailable'); + } + }); + disposals.push(() => db.persistence.close!()); + if (action !== 'insert') await seed(db.persistence); + const { rows, network } = await open(db.persistence, transport(), { + networkMode: 'manual', + }); + await rows.connect(); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + fail = true; + const transaction = + action === 'insert' + ? rows.insert({ id: document, title: 'local' }) + : action === 'update' + ? rows.update(document, (row) => { + row.title = 'local'; + }) + : rows.delete(document); + await expect(transaction.isPersisted.promise).rejects.toThrow('disk unavailable'); + expect( + network.client.mutation.mock.calls.some( + ([ref]: any) => getFunctionName(ref) === 'offline:replicate' + ) + ).toBe(false); + if (action === 'insert') { + expect( + db.database + .prepare("SELECT value FROM kv WHERE key = 'pending-create:offline:new'") + .get() + ).toBeDefined(); + } + fail = false; + await rows.disconnect(); + } +); + +test('rejected delete restores the durable document', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + await seed(db.persistence); + const network = transport(); + network.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate' && args.type === 'delete') { + throw new Error('server rejected delete'); + } + return { success: true, seq: 1 }; + }); + const { rows } = await open(db.persistence, network, { networkMode: 'manual' }); + await rows.connect(); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + const transaction = rows.delete('a'); + await expect(transaction.isPersisted.promise).rejects.toThrow('server rejected delete'); + const visible = rows.get('a')?.title; + disposals.pop(); + disposals.pop(); + db.database.close(); + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const next = await open(reopened.persistence, transport(), { networkMode: 'manual' }); + expect({ visible, reopened: next.rows.get('a')?.title }).toEqual({ + visible: 'persisted', + reopened: 'persisted', + }); +}); + +test('an update after a rejected delete converges with a server that rejected the delete delta', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + const base = await seed(db.persistence); + const server = new Y.Doc(); + Y.applyUpdateV2(server, base); + expect(server.getMap('fields').get('title')).toBe('persisted'); + const rejecting = transport(); + rejecting.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate' && args.type === 'delete') + throw new Error('server rejected delete'); + return { success: true, seq: 1 }; + }); + const { rows } = await open(db.persistence, rejecting, { networkMode: 'manual' }); + await rows.connect(); + await vi.waitFor(() => expect(rejecting.subscriptions.length).toBeGreaterThan(0)); + const deletion = rows.delete('a'); + await expect(deletion.isPersisted.promise).rejects.toThrow('server rejected delete'); + disposals.pop(); + disposals.pop(); + db.database.close(); + + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const accepting = transport(); + accepting.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate') + Y.applyUpdateV2(server, new Uint8Array(args.bytes)); + return { success: true, seq: 2 }; + }); + const next = await open(reopened.persistence, accepting, { networkMode: 'manual' }); + await next.rows.connect(); + const update = next.rows.update('a', (row) => { + row.title = 'after rollback'; + }); + await update.isPersisted.promise; + expect(server.getMap('fields').get('title')).toBe('after rollback'); + expect(server.getMap('_meta').get('_deleted')).not.toBe(true); + server.destroy(); +}); + +test('an older repair completion cannot clear a newer full-state repair', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + const base = await seed(db.persistence); + const server = new Y.Doc(); + Y.applyUpdateV2(server, base); + const first = deferred(); + const second = deferred(); + const updateCalls: any[] = []; + const network = transport(); + network.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) !== 'offline:replicate') return { success: true, seq: 1 }; + if (args.type === 'delete') throw new Error('server rejected delete'); + updateCalls.push(args); + return updateCalls.length === 1 ? first.promise : second.promise; + }); + const { rows } = await open(db.persistence, network, { networkMode: 'manual' }); + await rows.connect(); + const deletion = rows.delete('a'); + await expect(deletion.isPersisted.promise).rejects.toThrow('server rejected delete'); + expect(await db.persistence.kv.get('full-sync-required:offline:a')).toBeTypeOf('number'); + const older = rows.update('a', (row) => { + row.title = 'older'; + }); + await vi.waitFor(() => expect(updateCalls).toHaveLength(1)); + const newer = rows.update('a', (row) => { + row.title = 'newer'; + }); + await vi.waitFor(() => expect(updateCalls).toHaveLength(2)); + first.resolve({ success: true, seq: 2 }); + await older.isPersisted.promise; + expect(await db.persistence.kv.get('full-sync-required:offline:a')).toBeTypeOf('number'); + Y.applyUpdateV2(server, new Uint8Array(updateCalls[1].bytes)); + second.resolve({ success: true, seq: 3 }); + await newer.isPersisted.promise; + expect(await db.persistence.kv.get('full-sync-required:offline:a')).toBeUndefined(); + expect(server.getMap('fields').get('title')).toBe('newer'); + server.destroy(); +}); + +test('a newer rejected delete restores its durable repair marker after an older acknowledgement', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + const base = await seed(db.persistence); + const server = new Y.Doc(); + Y.applyUpdateV2(server, base); + let deletes = 0; + const network = transport(); + network.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) !== 'offline:replicate') return { success: true, seq: 1 }; + if (args.type === 'delete') { + deletes += 1; + throw new Error(`server rejected delete ${deletes}`); + } + Y.applyUpdateV2(server, new Uint8Array(args.bytes)); + return { success: true, seq: 2 }; + }); + const { rows } = await open(db.persistence, network, { networkMode: 'manual' }); + await rows.connect(); + const initialDelete = rows.delete('a'); + await expect(initialDelete.isPersisted.promise).rejects.toThrow('server rejected delete 1'); + + const entered = deferred(); + const release = deferred(); + const del = db.persistence.kv.del.bind(db.persistence.kv); + db.persistence.kv.del = async (key) => { + if (key === 'full-sync-required:offline:a') { + entered.resolve(); + await release.promise; + } + await del(key); + }; + const acceptedRepair = rows.update('a', (row) => { + row.title = 'accepted repair'; + }); + await entered.promise; + const newerDelete = rows.delete('a'); + const newerRejected = expect(newerDelete.isPersisted.promise).rejects.toThrow( + 'server rejected delete 2' + ); + await vi.waitFor(() => expect(deletes).toBe(2)); + release.resolve(); + await acceptedRepair.isPersisted.promise; + await newerRejected; + expect(await db.persistence.kv.get('full-sync-required:offline:a')).toBeTypeOf('number'); + disposals.pop(); + disposals.pop(); + db.database.close(); + + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const accepting = transport(); + accepting.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate') + Y.applyUpdateV2(server, new Uint8Array(args.bytes)); + return { success: true, seq: 3 }; + }); + const next = await open(reopened.persistence, accepting, { networkMode: 'manual' }); + await next.rows.connect(); + const finalUpdate = next.rows.update('a', (row) => { + row.title = 'after second rollback'; + }); + await finalUpdate.isPersisted.promise; + expect(server.getMap('fields').get('title')).toBe('after second rollback'); + server.destroy(); +}); + +test('rejected delete rollback preserves a newer streamed field value', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + await seed(db.persistence); + const remote = deferred(); + const network = transport(); + network.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate' && args.type === 'delete') { + return remote.promise; + } + return { success: true, seq: 1 }; + }); + const { rows } = await open(db.persistence, network, { networkMode: 'manual' }); + await rows.connect(); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + const transaction = rows.delete('a'); + const rejected = expect(transaction.isPersisted.promise).rejects.toThrow('server rejected delete'); + await vi.waitFor(() => + expect( + network.client.mutation.mock.calls.some( + ([ref, args]: any) => + getFunctionName(ref) === 'offline:replicate' && args.type === 'delete' + ) + ).toBe(true) + ); + network.emit(page(change('a', 2, 'concurrent'))); + await vi.waitFor(async () => expect(await db.persistence.kv.get('cursor:offline')).toBe(2)); + remote.reject(new Error('server rejected delete')); + await rejected; + expect(rows.get('a')?.title).toBe('concurrent'); + disposals.pop(); + disposals.pop(); + db.database.close(); + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const next = await open(reopened.persistence, transport(), { networkMode: 'manual' }); + expect(next.rows.get('a')?.title).toBe('concurrent'); +}); + +test('reset while delete rejection is pending does not resurrect the document', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + await seed(db.persistence); + const remote = deferred(); + const network = transport(); + network.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate' && args.type === 'delete') { + return remote.promise; + } + return { success: true, seq: 1 }; + }); + const { rows } = await open(db.persistence, network, { networkMode: 'manual' }); + await rows.connect(); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + const transaction = rows.delete('a'); + const rejected = expect(transaction.isPersisted.promise).rejects.toThrow('server rejected delete'); + await vi.waitFor(() => + expect( + network.client.mutation.mock.calls.some( + ([ref, args]: any) => + getFunctionName(ref) === 'offline:replicate' && args.type === 'delete' + ) + ).toBe(true) + ); + await rows.reset(); + remote.reject(new Error('server rejected delete')); + await rejected; + expect(rows.get('a')).toBeUndefined(); + disposals.pop(); + disposals.pop(); + db.database.close(); + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const next = await open(reopened.persistence, transport(), { networkMode: 'manual' }); + expect(next.rows.get('a')).toBeUndefined(); +}); + +test('an authoritative streamed deletion wins over a pending rejected delete', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + await seed(db.persistence); + const remote = deferred(); + const network = transport(); + network.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate' && args.type === 'delete') + return remote.promise; + return { success: true, seq: 1 }; + }); + const { rows } = await open(db.persistence, network, { networkMode: 'manual' }); + await rows.connect(); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + const transaction = rows.delete('a'); + const rejected = expect(transaction.isPersisted.promise).rejects.toThrow('server rejected delete'); + await vi.waitFor(() => + expect( + network.client.mutation.mock.calls.some( + ([ref, args]: any) => + getFunctionName(ref) === 'offline:replicate' && args.type === 'delete' + ) + ).toBe(true) + ); + network.emit(page({ document: 'a', seq: 2, type: 'delta', exists: false })); + await vi.waitFor(async () => expect(await db.persistence.kv.get('cursor:offline')).toBe(2)); + remote.reject(new Error('server rejected delete')); + await rejected; + expect(rows.get('a')).toBeUndefined(); + expect(await db.persistence.kv.get('full-sync-required:offline:a')).toBeUndefined(); + disposals.pop(); + disposals.pop(); + db.database.close(); + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const next = await open(reopened.persistence, transport(), { networkMode: 'manual' }); + expect(next.rows.get('a')).toBeUndefined(); +}); + +test('partial batch delete keeps the accepted item deleted and restores only the rejected item', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + await seed(db.persistence, 'a'); + await seed(db.persistence, 'b'); + let deletes = 0; + const network = transport(); + network.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate' && args.type === 'delete') { + deletes += 1; + if (deletes === 2) throw new Error('server rejected second delete'); + } + return { success: true, seq: deletes }; + }); + const { rows } = await open(db.persistence, network, { networkMode: 'manual' }); + await rows.connect(); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + const transaction = rows.delete(['a', 'b']); + await expect(transaction.isPersisted.promise).rejects.toThrow('server rejected second delete'); + expect(rows.get('a')).toBeUndefined(); + expect(rows.get('b')?.title).toBe('persisted'); + disposals.pop(); + disposals.pop(); + db.database.close(); + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const next = await open(reopened.persistence, transport(), { networkMode: 'manual' }); + expect(next.rows.get('a')).toBeUndefined(); + expect(next.rows.get('b')?.title).toBe('persisted'); +}); + +test('failed delete rollback persistence rejects explicitly and drains after the failure clears', async () => { + let failRollback = false; + const db = await disk(undefined, async (sql, params) => { + if ( + failRollback && + sql.startsWith('INSERT INTO deltas') && + params?.[0] === 'offline:a' + ) { + throw new Error('rollback disk unavailable'); + } + }); + disposals.push(() => db.persistence.close!()); + await seed(db.persistence); + const network = transport(); + network.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate' && args.type === 'delete') { + failRollback = true; + throw new Error('server rejected delete'); + } + return { success: true, seq: 1 }; + }); + const { rows } = await open(db.persistence, network, { networkMode: 'manual' }); + await rows.connect(); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + const transaction = rows.delete('a'); + await expect(transaction.isPersisted.promise).rejects.toThrow( + 'Delete failed and local rollback could not be persisted' + ); + expect( + db.database + .prepare("SELECT count(*) AS count FROM deltas WHERE collection = 'offline:a'") + .get() + ).toHaveProperty('count', 2); + failRollback = false; + await rows.disconnect(); + expect( + db.database + .prepare("SELECT count(*) AS count FROM deltas WHERE collection = 'offline:a'") + .get() + ).toHaveProperty('count', 3); + disposals.pop(); + disposals.pop(); + db.database.close(); + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const next = await open(reopened.persistence, transport(), { networkMode: 'manual' }); + expect(next.rows.get('a')?.title).toBe('persisted'); +}); + +test('failed delete rollback marker persistence rejects without publishing a restored row', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + await seed(db.persistence); + const set = db.persistence.kv.set.bind(db.persistence.kv); + db.persistence.kv.set = async (key, value) => { + if (key === 'full-sync-required:offline:a') throw new Error('repair marker unavailable'); + await set(key, value); + }; + const network = transport(); + network.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate' && args.type === 'delete') + throw new Error('server rejected delete'); + return { success: true, seq: 1 }; + }); + const { rows } = await open(db.persistence, network, { networkMode: 'manual' }); + await rows.connect(); + const deletion = rows.delete('a'); + await expect(deletion.isPersisted.promise).rejects.toThrow( + 'Delete failed and local rollback could not be persisted' + ); + expect(rows.get('a')).toBeUndefined(); + expect(await db.persistence.kv.get('full-sync-required:offline:a')).toBeUndefined(); +}); + +test('full-row updates remove omitted optional fields from outbound and reopened data', async () => { + const first = await disk(); + disposals.push(() => first.persistence.close!()); + const net = transport(); + const { rows } = await open(first.persistence, net, { networkMode: 'manual' }); + let seq = 0; + net.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate') + net.emit(page({ ...args, seq: ++seq, exists: true })); + return { success: true, seq }; + }); + await rows.connect(); + await rows.insert({ id: 'optional', title: 'row', note: 'remove me' }).isPersisted.promise; + await rows.update('optional', (row) => { + row.note = undefined; + }).isPersisted.promise; + await vi.waitFor(() => expect(rows.get('optional')).not.toHaveProperty('note')); + const update = net.client.mutation.mock.calls + .filter( + ([ref, args]: any) => getFunctionName(ref) === 'offline:replicate' && args.type === 'update' + ) + .at(-1)?.[1]; + expect(update.material).not.toHaveProperty('note'); + await rows.disconnect(); + await rows.cleanup(); + disposals.pop(); + await first.persistence.close!(); + + const reopened = await disk(first.path); + disposals.push(() => reopened.persistence.close!()); + const next = await open(reopened.persistence, transport(), { networkMode: 'manual' }); + expect(next.rows.get('optional')).toEqual({ id: 'optional', title: 'row' }); +}); + +test('delayed data and cursor writes cannot be overtaken by marks or concurrent callbacks', async () => { + const dataGate = deferred(); + const cursorGate = deferred(); + let delayData = true; + let delayCursor = true; + const db = await disk(undefined, async (sql) => { + if (sql.startsWith('INSERT INTO deltas') && delayData) await dataGate.promise; + if (sql.startsWith('INSERT OR REPLACE INTO kv') && delayCursor) await cursorGate.promise; + }); + disposals.push(() => db.persistence.close!()); + // Client id is a kv write; allow initialization before delaying cursor specifically. + delayCursor = false; + const { rows, network } = await open(db.persistence); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + delayCursor = true; + const callback = network.subscriptions[0].callback; + callback(page(change('a', 1))); + callback(page(change('a', 1), change('b', 2))); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect( + db.database.prepare("SELECT value FROM kv WHERE key = 'cursor:offline'").get() + ).toBeUndefined(); + expect(network.client.mutation).not.toHaveBeenCalled(); + delayData = false; + dataGate.resolve(); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect( + db.database.prepare("SELECT value FROM kv WHERE key = 'cursor:offline'").get() + ).toBeUndefined(); + expect(network.client.mutation).not.toHaveBeenCalled(); + delayCursor = false; + cursorGate.resolve(); + await vi.waitFor(() => expect(rows.get('b')).toBeDefined()); + await vi.waitFor(async () => expect(await db.persistence.kv.get('cursor:offline')).toBe(2)); +}); + +test.each(['data', 'cursor'])( + 'failed %s writes retain progress and replay successfully', + async (failure) => { + let fail = false; + const db = await disk(undefined, async (sql, params) => { + if ( + fail && + ((failure === 'data' && sql.startsWith('INSERT INTO deltas')) || + (failure === 'cursor' && sql.startsWith('INSERT') && params?.[0] === 'cursor:offline')) + ) + throw new Error('disk unavailable'); + }); + disposals.push(() => db.persistence.close!()); + const { rows, network } = await open(db.persistence); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + fail = true; + network.emit(page(change('a', 1))); + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(await db.persistence.kv.get('cursor:offline')).toBeUndefined(); + expect(network.client.mutation).not.toHaveBeenCalled(); + fail = false; + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(1), { + timeout: 2000, + }); + network.emit(page(change('a', 1))); + await vi.waitFor(() => expect(rows.get('a')?.title).toBe('remote')); + await vi.waitFor(async () => expect(await db.persistence.kv.get('cursor:offline')).toBe(1)); + } +); + +test('reset drains a delayed write and stale callbacks cannot resurrect disk data', async () => { + const gate = deferred(); + let writing = false; + const db = await disk(undefined, async (sql) => { + if (sql.startsWith('INSERT INTO deltas')) { + writing = true; + await gate.promise; + } + }); + const { rows, network } = await open(db.persistence); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + const callback = network.subscriptions[0].callback; + callback(page(change('a', 1))); + await vi.waitFor(() => expect(writing).toBe(true)); + let done = false; + const reset = rows.reset().then(() => { + done = true; + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(done).toBe(false); + gate.resolve(); + await reset; + callback(page(change('b', 2))); + expect(await db.persistence.listDocuments('offline')).toEqual([]); + expect(await db.persistence.kv.get('cursor:offline')).toBeUndefined(); + await rows.cleanup(); + disposals.pop(); + await db.persistence.close!(); + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + expect(await reopened.persistence.listDocuments('offline')).toEqual([]); +}); + +test('old recovery absence cannot remove a more recent stream version', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + await seed(db.persistence); + const recovery = deferred(); + const net = transport(); + net.client.query.mockImplementation(() => recovery.promise); + const { rows } = await open(db.persistence, net); + await vi.waitFor(() => expect(net.subscriptions.length).toBeGreaterThan(0)); + net.emit(page(change('a', 1, 'newer'))); + await vi.waitFor(() => expect(awaitableTitle(rows)).toBe('newer')); + recovery.resolve({ mode: 'recovery', exists: false }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(rows.get('a')?.title).toBe('newer'); +}); +const awaitableTitle = (rows: any) => rows.get('a')?.title; + +test('account generation rejects stale recovery and uses newly captured request context', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + await seed(db.persistence); + const recovery = deferred(); + const net = transport(); + net.client.query.mockImplementation(() => recovery.promise); + let guest = 'first'; + const { rows } = await open(db.persistence, net, { + networkMode: 'manual', + requestArgs: () => ({ guest }), + }); + await rows.connect(); + await rows.disconnect(); + guest = 'second'; + net.client.query.mockResolvedValue({ mode: 'recovery', exists: true }); + await rows.connect(); + recovery.resolve({ mode: 'recovery', exists: false }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(rows.get('a')).toBeDefined(); + expect((net.client.query.mock.calls.at(-1) as any)[1].guest).toBe('second'); +}); + +test.each(['seq', 'document', 'interval', 'threshold', 'exclude'])( + 'request context rejects reserved %s', + (key) => { + const net = transport(); + const connection = createConnection(net.client as any, { requestArgs: () => ({ [key]: 1 }) }); + expect(() => connection.connect()).toThrow('protocol argument'); + expect(net.client.query).not.toHaveBeenCalled(); + } +); + +test('raw incremental update survives disk restart before its compacted base arrives', async () => { + const original = new Y.Doc(); + original.getMap('fields').set('title', 'base'); + const base = Y.encodeStateAsUpdateV2(original); + const vector = Y.encodeStateVector(original); + original.getMap('fields').set('title', 'latest'); + const delta = Y.encodeStateAsUpdateV2(original, vector); + original.destroy(); + const db = await disk(); + const { rows, network } = await open(db.persistence); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + network.emit(page({ document: 'a', seq: 1, type: 'delta', exists: true, bytes: delta.buffer })); + await vi.waitFor(async () => expect(await db.persistence.kv.get('cursor:offline')).toBe(1)); + await rows.disconnect(); + await rows.cleanup(); + disposals.pop(); + await db.persistence.close!(); + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const next = await open(reopened.persistence); + await vi.waitFor(() => expect(next.network.subscriptions.length).toBeGreaterThan(0)); + next.network.emit( + page({ document: 'a', seq: 2, type: 'snapshot', exists: true, bytes: base.buffer }) + ); + await vi.waitFor(() => expect(next.rows.get('a')?.title).toBe('latest')); +}); + +test('SQLite clear rolls back both tables, fences writes, and survives reopen', async () => { + const first = await disk(); + const source = new Y.Doc(); + source.getMap('fields').set('title', 'base'); + const base = Y.encodeStateAsUpdate(source); + const vector = Y.encodeStateVector(source); + source.getMap('fields').set('title', 'complete'); + const delta = Y.encodeStateAsUpdate(source, vector); + first.database + .prepare('INSERT INTO snapshots (collection, data) VALUES (?, ?)') + .run('offline:a', base); + first.database + .prepare('INSERT INTO deltas (collection, data) VALUES (?, ?)') + .run('offline:a', delta); + first.database.exec(` + CREATE TRIGGER reject_delta_delete + BEFORE DELETE ON deltas + BEGIN SELECT RAISE(ABORT, 'blocked delta delete'); END + `); + const loaded = new Y.Doc(); + const provider = first.persistence.createDocPersistence('offline:a', loaded); + await provider.whenSynced; + expect(loaded.getMap('fields').get('title')).toBe('complete'); + await expect(provider.clear!()).rejects.toThrow('blocked delta delete'); + expect( + first.database + .prepare("SELECT count(*) AS count FROM snapshots WHERE collection = 'offline:a'") + .get() + ).toHaveProperty('count', 1); + expect( + first.database + .prepare("SELECT count(*) AS count FROM deltas WHERE collection = 'offline:a'") + .get() + ).toHaveProperty('count', 1); + first.database.exec('DROP TRIGGER reject_delta_delete'); + await first.persistence.close!(); + + const reopened = await disk(first.path); + const restored = new Y.Doc(); + const retry = reopened.persistence.createDocPersistence('offline:a', restored); + await retry.whenSynced; + expect(restored.getMap('fields').get('title')).toBe('complete'); + await retry.clear!(); + await expect(retry.storeUpdate!(base)).rejects.toThrow('closed'); + await reopened.persistence.close!(); + + const final = await disk(first.path); + disposals.push(() => final.persistence.close!()); + expect(await final.persistence.listDocuments('offline')).toEqual([]); + source.destroy(); + loaded.destroy(); + restored.destroy(); +}); + +test('SQLite close releases the database after a queued write fails', async () => { + const first = await disk(undefined, async (sql) => { + if (sql.startsWith('INSERT INTO deltas')) throw new Error('injected write failure'); + }); + const doc = new Y.Doc(); + const provider = first.persistence.createDocPersistence('offline:a', doc); + await provider.whenSynced; + doc.getMap('fields').set('title', 'failure'); + await expect(provider.flush!()).rejects.toThrow('injected write failure'); + await expect(first.persistence.close!()).rejects.toThrow('injected write failure'); + expect(first.isClosed()).toBe(true); + const reopened = await disk(first.path); + disposals.push(() => reopened.persistence.close!()); + expect(await reopened.persistence.listDocuments('offline')).toEqual([]); + doc.destroy(); +}); + +test('custom adapter drains delayed updates before durable removal', async () => { + const values = new Map(); + const gate = deferred(); + const p = createCustomPersistence({ + async get(key) { + return values.get(key); + }, + async keys(prefix) { + return [...values.keys()].filter((k) => k.startsWith(prefix)); + }, + async set(key, value) { + await gate.promise; + values.set(key, value); + }, + async delete(key) { + values.delete(key); + }, + }); + const doc = new Y.Doc(); + const provider = p.createDocPersistence('offline:a', doc); + await provider.whenSynced; + doc.getMap('fields').set('title', 'delayed'); + const clear = provider.clear!(); + gate.resolve(); + await clear; + const next = p.createDocPersistence('offline:a', new Y.Doc()); + await next.whenSynced; + expect(await p.listDocuments('offline')).toEqual([]); + provider.destroy(); + next.destroy(); + doc.destroy(); +}); + +test('a later disconnect wins over connect waiting for initialization', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + const net = transport(); + const gate = deferred(); + const original = db.persistence.listDocuments; + db.persistence.listDocuments = async (prefix) => { + await gate.promise; + return original(prefix); + }; + const lazy = collection.create<{ id: string; title: string }>({ + schema: definition, + persistence: async () => db.persistence, + config: () => ({ convexClient: net.client as any, api, getKey: (row) => row.id }), + }); + await lazy.init(); + const rows = lazy.get(); + const loading = rows.preload(); + const connecting = rows.connect(); + const closing = rows.disconnect(); + gate.resolve(); + await Promise.all([loading, connecting, closing]); + expect(net.client.query).not.toHaveBeenCalled(); + expect(net.client.onUpdate).not.toHaveBeenCalled(); + await rows.cleanup(); +}); + +test('connect waits for reset data removal before starting the next generation', async () => { + const gate = deferred(); + let deleting = false; + const db = await disk(undefined, async (sql) => { + if (sql.startsWith('DELETE FROM snapshots')) { + deleting = true; + await gate.promise; + } + }); + disposals.push(() => db.persistence.close!()); + await seed(db.persistence); + const { rows, network } = await open(db.persistence); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + const reset = rows.reset(); + await vi.waitFor(() => expect(deleting).toBe(true)); + const previous = network.subscriptions.length; + const connecting = rows.connect(); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(network.subscriptions.length).toBe(previous); + gate.resolve(); + await Promise.all([reset, connecting]); + expect(network.subscriptions.at(-1)?.args.seq).toBe(0); + expect(rows.get('a')).toBeUndefined(); +}); + +test('writable offline creation survives restart and retries as insert after authorized connection', async () => { + const db = await disk(); + const { rows } = await open(db.persistence, transport(), { networkMode: 'manual' }); + await expect( + rows.insert({ id: 'offline-new', title: 'unsent' }).isPersisted.promise + ).rejects.toThrow('disconnected'); + await rows.disconnect(); + await rows.cleanup(); + disposals.pop(); + await db.persistence.close!(); + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const net = transport(); + net.client.query.mockResolvedValue({ mode: 'recovery', exists: false, known: false }); + const next = await open(reopened.persistence, net, { networkMode: 'manual' }); + expect(next.rows.get('offline-new')?.title).toBe('unsent'); + await next.rows.connect(); + await vi.waitFor(() => + expect(net.client.mutation.mock.calls.some((call: any) => call[1].type === 'insert')).toBe(true) + ); + expect(next.rows.get('offline-new')?.title).toBe('unsent'); +}); + +test('presence and marks exclude local edits whose writes are still delayed', async () => { + const cursorGate = deferred(); + const localGate = deferred(); + let blockCursor = false; + let blockLocal = false; + const db = await disk(undefined, async (sql, params) => { + if (blockCursor && sql.startsWith('INSERT') && params?.[0] === 'cursor:offline') + await cursorGate.promise; + if (blockLocal && sql.startsWith('INSERT INTO deltas')) await localGate.promise; + }); + disposals.push(() => db.persistence.close!()); + const { rows, network } = await open(db.persistence); + await vi.waitFor(() => expect(network.subscriptions.length).toBeGreaterThan(0)); + network.emit(page(change('a', 1))); + await vi.waitFor(() => expect(rows.get('a')).toBeDefined()); + const before = network.client.mutation.mock.calls.find( + (call: any) => call[1].action === 'mark' + ) as any; + const durable = new Uint8Array(before[1].vector); + blockCursor = true; + network.emit(page(change('b', 2))); + await new Promise((resolve) => setTimeout(resolve, 20)); + blockLocal = true; + const update = rows.update('a', (row) => { + row.title = 'not durable'; + }); + const handle = rows.doc('a'); + handle.presence.join(); + cursorGate.resolve(); + await vi.waitFor(() => + expect(network.client.mutation.mock.calls.some((call: any) => call[1].action === 'join')).toBe( + true + ) + ); + const join = network.client.mutation.mock.calls.find( + (call: any) => call[1].action === 'join' + ) as any; + expect(new Uint8Array(join[1].vector)).toEqual(durable); + localGate.resolve(); + await update.isPersisted.promise; + await rows.disconnect(); +}); + +test('encrypted raw updates survive compaction, disk reopen, and durable removal', async () => { + const db = await disk(); + const config = (storage: Persistence) => ({ + storage, + user: 'test', + unlock: { + passphrase: { + setup: async () => 'test-only-password', + get: async () => 'test-only-password', + }, + }, + }); + const encrypted = await createWebEncryptionPersistence(config(db.persistence)); + await encrypted.unlock(); + const local = new Y.Doc(); + const provider = encrypted.createDocPersistence('offline:a', local); + await provider.whenSynced; + const server = new Y.Doc(); + server.getMap('fields').set('title', 'base'); + const base = Y.encodeStateAsUpdateV2(server); + for (let index = 0; index < 50; index++) { + const vector = Y.encodeStateVector(server); + server.getMap('fields').set('title', `value-${index}`); + const update = Y.encodeStateAsUpdateV2(server, vector); + await provider.storeUpdate!(update); + Y.applyUpdateV2(local, update, 'replicate-persisted'); + } + provider.destroy(); + await db.persistence.close!(); + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const next = await createWebEncryptionPersistence(config(reopened.persistence)); + await next.unlock(); + expect(await next.listDocuments('offline')).toEqual(['a']); + const restored = new Y.Doc(); + const nextProvider = next.createDocPersistence('offline:a', restored); + await nextProvider.whenSynced; + await nextProvider.storeUpdate!(base); + Y.applyUpdateV2(restored, base, 'replicate-persisted'); + expect(restored.getMap('fields').get('title')).toBe('value-49'); + await nextProvider.clear!(); + expect(await next.listDocuments('offline')).toEqual([]); + server.destroy(); + local.destroy(); + restored.destroy(); +}); + +test('an insert before preload is not invalidated by initial automatic activation', async () => { + const db = await disk(); + disposals.push(() => db.persistence.close!()); + const net = transport(); + const lazy = collection.create<{ id: string; title: string }>({ + schema: definition, + persistence: async () => db.persistence, + config: () => ({ convexClient: net.client as any, api, getKey: (row) => row.id }), + }); + await lazy.init(); + const rows = lazy.get(); + const transaction = rows.insert({ id: 'early', title: 'accepted' }); + await rows.preload(); + await transaction.isPersisted.promise; + expect(net.client.mutation.mock.calls.some((call: any) => call[1].type === 'insert')).toBe(true); + await rows.disconnect(); + await rows.cleanup(); +}); + +test('repair content sync pairs current full bytes with current material after a stream update', async () => { + const gateEntered = deferred(); + const releaseGate = deferred(); + let gateWrites = false; + let gateConsumed = false; + const db = await disk(); + disposals.push(() => db.persistence.close!()); + const base = await seedProse(db.persistence); + const server = new Y.Doc(); + Y.applyUpdateV2(server, base); + const network = transport(); + const accepted: any[] = []; + network.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) !== 'offline:replicate') return { success: true, seq: 1 }; + if (args.type === 'delete') throw new Error('server rejected delete'); + accepted.push(args); + return { success: true, seq: 3 }; + }); + const gatedPersistence: Persistence = { + ...db.persistence, + createDocPersistence(name, doc) { + const provider = db.persistence.createDocPersistence(name, doc); + const flush = provider.flush!.bind(provider); + provider.flush = async () => { + if (gateWrites && !gateConsumed && name === 'offline:a') { + gateConsumed = true; + await flush(); + gateEntered.resolve(); + await releaseGate.promise; + return; + } + await flush(); + }; + return provider; + }, + }; + const { rows } = await openProse(gatedPersistence, network); + await rows.connect(); + const deletion = rows.delete('a'); + await expect(deletion.isPersisted.promise).rejects.toThrow('server rejected delete'); + expect(await db.persistence.kv.get('full-sync-required:offline:a')).toBeTypeOf('number'); + + const contentCaptured = deferred(); + const update = rows.update.bind(rows); + (rows as any).update = (...args: any[]) => { + if (args[1]?.metadata?.contentSync) contentCaptured.resolve(); + return update(...args); + }; + const binding = await rows.doc('a').prose('content', { debounceMs: 0 }); + gateWrites = true; + const text = new Y.XmlText(); + text.insert(0, 'local prose'); + binding.fragment.insert(0, [text]); + await gateEntered.promise; + await contentCaptured.promise; + network.emit(proseTitleChange(base, 'remote', 2)); + await vi.waitFor(() => + expect(binding.provider.document.getMap('fields').get('title')).toBe('remote') + ); + releaseGate.resolve(); + await vi.waitFor(() => expect(accepted).toHaveLength(1)); + + const sent = accepted[0]; + Y.applyUpdateV2(server, new Uint8Array(sent.bytes)); + expect(server.getMap('_meta').get('_deleted')).not.toBe(true); + expect(server.getMap('fields').get('title')).toBe('remote'); + expect((server.getMap('fields').get('content') as Y.XmlFragment).toString()).toBe('local prose'); + expect(sent.material).toEqual(proseMaterial(server)); + await vi.waitFor(async () => + expect(await db.persistence.kv.get('full-sync-required:offline:a')).toBeUndefined() + ); + + binding.destroy(); + disposals.pop(); + disposals.pop(); + db.database.close(); + const reopened = await disk(db.path); + disposals.push(() => reopened.persistence.close!()); + const next = await openProse(reopened.persistence); + expect(next.rows.get('a')).toMatchObject({ title: 'remote', content: sent.material.content }); + expect(proseMaterial(server)).toEqual(sent.material); + server.destroy(); +}, 20_000); + +test('restart leaves a repair marker idle until an explicit update sends full state', async () => { + const first = await disk(); + disposals.push(() => first.persistence.close!()); + const base = await seedProse(first.persistence); + const rejecting = transport(); + rejecting.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate' && args.type === 'delete') + throw new Error('server rejected delete'); + return { success: true, seq: 1 }; + }); + const initial = await openProse(first.persistence, rejecting); + await initial.rows.connect(); + const deletion = initial.rows.delete('a'); + await expect(deletion.isPersisted.promise).rejects.toThrow('server rejected delete'); + const marker = await first.persistence.kv.get('full-sync-required:offline:a'); + expect(marker).toBeTypeOf('number'); + disposals.pop(); + disposals.pop(); + first.database.close(); + + const reopened = await disk(first.path); + disposals.push(() => reopened.persistence.close!()); + const server = new Y.Doc(); + Y.applyUpdateV2(server, base); + const recoveryVector = Y.encodeStateVector(server); + server.getMap('fields').set('title', 'recovered'); + const recoveryDiff = Y.encodeStateAsUpdateV2(server, recoveryVector); + const accepting = transport(); + accepting.client.query.mockResolvedValue({ + mode: 'recovery', + exists: true, + diff: recoveryDiff.buffer, + }); + const accepted: any[] = []; + accepting.client.mutation.mockImplementation(async (ref: any, args: any) => { + if (getFunctionName(ref) === 'offline:replicate') accepted.push(args); + return { success: true, seq: 2 }; + }); + const next = await openProse(reopened.persistence, accepting); + await next.rows.connect(); + await vi.waitFor(() => expect(accepting.subscriptions.length).toBeGreaterThan(0)); + await vi.waitFor(() => expect(next.rows.get('a')?.title).toBe('recovered')); + expect(accepted).toHaveLength(0); + expect(await reopened.persistence.kv.get('full-sync-required:offline:a')).toBe(marker); + + const update = next.rows.update('a', (row) => { + row.title = 'explicit'; + }); + await update.isPersisted.promise; + expect(accepted).toHaveLength(1); + expect(accepted[0].type).toBe('update'); + Y.applyUpdateV2(server, new Uint8Array(accepted[0].bytes)); + expect(proseMaterial(server)).toEqual(accepted[0].material); + expect(server.getMap('fields').get('title')).toBe('explicit'); + expect(await reopened.persistence.kv.get('full-sync-required:offline:a')).toBeUndefined(); + server.destroy(); +}); diff --git a/packages/replicate/src/test/persistence-writes.test.ts b/packages/replicate/src/test/persistence-writes.test.ts new file mode 100644 index 0000000..de9552f --- /dev/null +++ b/packages/replicate/src/test/persistence-writes.test.ts @@ -0,0 +1,22 @@ +import { expect, test } from 'vitest'; +import { PersistenceWrites } from '../client/persistence/writes'; + +test('flush drains writes added at the previous drain completion boundary', async () => { + const writes = new PersistenceWrites(); + const events: string[] = []; + let capturedFlush: Promise | undefined; + writes.add(async () => { + events.push('first'); + queueMicrotask(() => + queueMicrotask(() => { + writes.add(async () => { + events.push('second'); + }); + capturedFlush = writes.flush(); + }) + ); + }); + await writes.flush(); + await capturedFlush; + expect(events).toEqual(['first', 'second']); +}); diff --git a/packages/replicate/src/test/recovery.test.ts b/packages/replicate/src/test/recovery.test.ts new file mode 100644 index 0000000..a5eac62 --- /dev/null +++ b/packages/replicate/src/test/recovery.test.ts @@ -0,0 +1,89 @@ +import { expect, test } from 'vitest'; +import { convexTest } from 'convex-test'; +import { defineSchema, defineTable, makeFunctionReference } from 'convex/server'; +import { v } from 'convex/values'; +import * as Y from 'yjs'; +import componentSchema from '../component/schema'; +import { modules } from '../component/test.setup'; +const appSchema = defineSchema({ + records: defineTable({ + id: v.string(), + title: v.string(), + timestamp: v.optional(v.number()), + }).index('by_doc_id', ['id']), +}); +const appModules = { + './_generated/server.ts': async () => ({}), + './functions.ts': () => import('./server-functions'), +}; + +test('actual server recovery distinguishes absent material from unknown component history', async () => { + const t = convexTest(appSchema, appModules); + t.registerComponent('replicate', componentSchema, modules); + const doc = new Y.Doc(); + doc.getMap('fields').set('title', 'record'); + const vector = Y.encodeStateVector(new Y.Doc()).buffer as ArrayBuffer; + const delta = makeFunctionReference<'query'>('functions:delta'); + const missing = await t.query(delta, { document: 'missing', vector }); + expect(missing).toMatchObject({ exists: false, known: false }); + const id = await t.run((ctx) => ctx.db.insert('records', { id: 'a', title: 'record' })); + // Access component through the same generated component reference as application functions. + await t.run(async (ctx) => { + const { componentsGeneric } = await import('convex/server'); + await ctx.runMutation((componentsGeneric() as any).replicate.mutations.insertDocument, { + collection: 'records', + document: 'a', + bytes: Y.encodeStateAsUpdateV2(doc).buffer as ArrayBuffer, + }); + }); + expect(await t.query(delta, { document: 'a', vector })).toMatchObject({ + exists: true, + known: true, + }); + await t.run((ctx) => ctx.db.delete(id)); + expect(await t.query(delta, { document: 'a', vector })).toMatchObject({ + exists: false, + known: true, + }); + doc.destroy(); +}); + +test('retried create after an earlier commit does not duplicate logical records', async () => { + const t = convexTest(appSchema, appModules); + t.registerComponent('replicate', componentSchema, modules); + const doc = new Y.Doc(); + doc.getMap('fields').set('title', 'once'); + const args = { + document: 'retry', + bytes: Y.encodeStateAsUpdateV2(doc).buffer as ArrayBuffer, + material: { id: 'retry', title: 'once' }, + type: 'insert', + }; + const mutation = makeFunctionReference<'mutation'>('functions:replicate'); + const absent = await t.query(makeFunctionReference<'query'>('functions:delta'), { + document: 'retry', + vector: Y.encodeStateVector(new Y.Doc()).buffer as ArrayBuffer, + }); + expect(absent.exists).toBe(false); + const original = await t.mutation(mutation, args); + doc.getMap('fields').set('title', 'newer offline edit'); + const retry = await t.mutation(mutation, { + ...args, + bytes: Y.encodeStateAsUpdateV2(doc).buffer as ArrayBuffer, + material: { id: 'retry', title: 'newer offline edit' }, + }); + expect(retry.seq).toBeGreaterThan(original.seq); + const recovered = await t.query(makeFunctionReference<'query'>('functions:delta'), { + document: 'retry', + vector: Y.encodeStateVector(new Y.Doc()).buffer as ArrayBuffer, + }); + const merged = new Y.Doc(); + Y.applyUpdateV2(merged, new Uint8Array(recovered.diff)); + expect(merged.getMap('fields').get('title')).toBe('newer offline edit'); + merged.destroy(); + expect(await t.run((ctx) => ctx.db.query('records').first())).toMatchObject({ + title: 'newer offline edit', + }); + expect(await t.run((ctx) => ctx.db.query('records').collect())).toHaveLength(1); + doc.destroy(); +}); diff --git a/packages/replicate/src/test/server-functions.ts b/packages/replicate/src/test/server-functions.ts new file mode 100644 index 0000000..0650bbf --- /dev/null +++ b/packages/replicate/src/test/server-functions.ts @@ -0,0 +1,6 @@ +import { componentsGeneric } from 'convex/server'; +import { Replicate } from '../server'; +const storage = new Replicate((componentsGeneric() as any).replicate, 'records'); +export const delta = storage.createDeltaQuery(); + +export const replicate = storage.createReplicateMutation(); diff --git a/packages/replicate/test/browser/baseline-wa-sqlite-manifest.json b/packages/replicate/test/browser/baseline-wa-sqlite-manifest.json new file mode 100644 index 0000000..64d67a8 --- /dev/null +++ b/packages/replicate/test/browser/baseline-wa-sqlite-manifest.json @@ -0,0 +1,17 @@ +{ + "repository": "https://github.com/rhashimoto/wa-sqlite", + "revision": "55bf0b6646ac1bb81b00b9c25636b59a87e4a918", + "buildRevision": "44cc640c65f29de9bfbc2004974c9f6c2c690c2b", + "sha256": { + "dist/wa-sqlite-async.mjs": "0ba7e66b84408267ac00a5f75b1f3c733f258fbc6b6dc91baa567e0e963892a9", + "dist/wa-sqlite-async.wasm": "c3af64afbe6034af23fb756bb040e7d8d73aaf6596f9f2b3ea463c8c036bf5e5", + "src/examples/IDBBatchAtomicVFS.js": "0b72ab6b1a692c5e5d56a0dd387c3de5a65265c91a0da8baf90407a7f0dd043e", + "src/FacadeVFS.js": "dd00dc59720c0bebb91b6f982e5339351997467ff868a608e915cc020e13fbca", + "src/VFS.js": "55e7455e2ec0000cc4ba7a8bf7882f414a40f2a0ba6eb4c4ac847f379f54d92f", + "src/WebLocksMixin.js": "1ea5c7e9e02dc94566b910b396f17a3c93aeab04a208177e8c1bd3bb65618d44", + "src/sqlite-api.js": "8d7a59fa1df49859e584561945307b61460d3b544e10f9d2cb2514aef1ecfd42", + "src/sqlite-constants.js": "f7b570f0c39e4b54c99f2598b6ad42018d4d8bf791861ede20d295628ae49f8f", + "LICENSE": "547f9e97e461bd369cc64b6fc12d724e8b02764b8db7812691f29f0ec572be79", + "ACKNOWLEDGMENTS.md": "5dae12fa0c82fe1a6a70fcf6b765298b5f32b7d32449f4d72ae1c12468457762" + } +} diff --git a/packages/replicate/test/browser/index.html b/packages/replicate/test/browser/index.html new file mode 100644 index 0000000..d91a502 --- /dev/null +++ b/packages/replicate/test/browser/index.html @@ -0,0 +1,2 @@ +Offline replica regression + diff --git a/packages/replicate/test/browser/main.ts b/packages/replicate/test/browser/main.ts new file mode 100644 index 0000000..ee1e766 --- /dev/null +++ b/packages/replicate/test/browser/main.ts @@ -0,0 +1,109 @@ +import { collection, persistence } from '@trestleinc/replicate/client'; +import { schema } from '@trestleinc/replicate/server'; +import { makeFunctionReference } from 'convex/server'; +import { v } from 'convex/values'; +import * as Y from 'yjs'; +import Worker from '@trestleinc/replicate/worker?worker'; + +const api = Object.fromEntries( + ['material', 'delta', 'replicate', 'presence', 'session'].map((name) => [ + name, + makeFunctionReference(`browser:${name}`), + ]) +); +let callback: ((page: unknown) => void) | undefined; +let calls = 0; +let absent = false; +const transport = { + query: async () => { + calls++; + return { mode: 'recovery', exists: !absent, diff: null }; + }, + mutation: async () => { + calls++; + return null; + }, + onUpdate: (_api: unknown, _args: unknown, cb: typeof callback) => { + calls++; + callback = cb; + return () => { + callback = undefined; + }; + }, +}; +let closeMessages = 0; +const sqliteWorker = new Worker(); +const postMessage = sqliteWorker.postMessage.bind(sqliteWorker); +sqliteWorker.postMessage = ((message: { type?: number }, options?: StructuredSerializeOptions) => { + if (message.type === 2) closeMessages++; + postMessage(message, options); +}) as typeof sqliteWorker.postMessage; +const runtime = new URLSearchParams(location.hash.slice(1)); +const local = await persistence.web.sqlite.create({ + name: runtime.get('database') ?? 'offline-browser', + worker: sqliteWorker, + assetBase: runtime.get('assetBase') ?? '/sqlite', +}); +const lazy = collection.create<{ id: string; title: string }>({ + schema: schema.define({ + shape: v.object({ id: v.string(), title: v.string() }), + defaults: {}, + history: {}, + }), + persistence: async () => local, + config: () => ({ convexClient: transport as any, api: api as any, getKey: (row) => row.id }), + networkMode: 'manual', + readOnly: true, +}); +await lazy.init(); +const rows = lazy.get(); +await rows.preload(); +Object.assign(window, { + replica: { + get: (id: string) => rows.get(id), + calls: () => calls, + connect: () => rows.connect(), + disconnect: () => rows.disconnect(), + reset: () => rows.reset(), + absent: () => { + absent = true; + }, + cursor: () => local.kv.get('cursor:browser'), + close: async () => { + await rows.disconnect(); + await rows.cleanup(); + await local.close!(); + }, + closeRace: async () => { + await rows.disconnect(); + await rows.cleanup(); + let acceptedCompleted = false; + const accepted = local.kv.set('accepted-before-close', true).then(() => { + acceptedCompleted = true; + }); + const first = local.close!(); + const second = local.close!(); + let lateRejected = false; + try { + await local.kv.get('late'); + } catch { + lateRejected = true; + } + await Promise.all([accepted, first, second]); + return { acceptedCompleted, closeMessages, lateRejected }; + }, + emit: (id: string, seq: number, exists = true, raw?: number[]) => { + const doc = new Y.Doc(); + doc.getMap('fields').set('title', 'persisted in browser'); + const bytes = raw ? new Uint8Array(raw) : Y.encodeStateAsUpdateV2(doc); + doc.destroy(); + callback?.({ + mode: 'stream', + changes: [{ document: id, seq, bytes: bytes.buffer, exists, type: 'snapshot' }], + seq, + }); + }, + }, +}); +await navigator.serviceWorker.register('/sw.js'); +await navigator.serviceWorker.ready; diff --git a/packages/replicate/test/native/index.js b/packages/replicate/test/native/index.js new file mode 100644 index 0000000..bd3236f --- /dev/null +++ b/packages/replicate/test/native/index.js @@ -0,0 +1,2 @@ +import { collection, persistence } from '@trestleinc/replicate/client'; +globalThis.replicateNativeGraph = { collection, persistence }; diff --git a/packages/replicate/test/native/package.json b/packages/replicate/test/native/package.json new file mode 100644 index 0000000..3dd2232 --- /dev/null +++ b/packages/replicate/test/native/package.json @@ -0,0 +1,5 @@ +{ + "name": "replicate-native-graph", + "private": true, + "main": "index.js" +} diff --git a/packages/replicate/tsdown.config.ts b/packages/replicate/tsdown.config.ts index 0fec46a..661d349 100644 --- a/packages/replicate/tsdown.config.ts +++ b/packages/replicate/tsdown.config.ts @@ -36,8 +36,6 @@ export default defineConfig([ '@logtape/logtape', 'effect', '@effect/platform', - 'wa-sqlite', - /^wa-sqlite/, '@op-engineering/op-sqlite', 'abstract-level', 'browser-level', @@ -54,7 +52,7 @@ export default defineConfig([ clean: false, platform: 'browser', shims: true, - external: ['yjs', /^wa-sqlite/], + external: ['yjs'], outExtensions: jsExtensions, }, {