fix: make offline read replicas durable across restart and compaction

.gitattributes

@@ -0,0 +1 @@
+packages/replicate/assets/wa-sqlite/** -text -whitespace linguist-vendored

.github/workflows/ci.yml

@@ -6,25 +6,47 @@ on:
   pull_request:
     branches: [main]
 
+permissions:
+  contents: read
+
 jobs:
   ci:
     name: Lint & Type Check
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: 24
 
       - name: Setup Bun
-        uses: oven-sh/setup-bun@v2
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
 
       - name: Install dependencies
         run: bun install --frozen-lockfile
 
       - name: Build packages
         run: bun run build:packages
 
+      - name: Test replication contracts
+        run: bun run test
+
+      - name: Verify SQLite assets
+        run: bun run --filter @trestleinc/replicate test:assets
+
+      - name: Install Chromium
+        working-directory: packages/replicate
+        run: bunx playwright install --with-deps chromium
+
+      - name: Test offline browser restarts
+        run: bun run --filter @trestleinc/replicate test:browser
+
       - name: Lint
         run: bun run lint
 
       - name: Type check
+        env:
+          PUBLIC_CONVEX_URL: https://example.convex.cloud
         run: bun run check

.gitignore

@@ -6,6 +6,9 @@
 # Dist
 node_modules
 dist/
+# Versioned upstream runtime distribution (not this repository's build output).
+!packages/replicate/assets/wa-sqlite/dist/
+!packages/replicate/assets/wa-sqlite/dist/*
 packages/*/dist
 apps/*/dist
 apps/*/.env

.prettierignore

@@ -5,3 +5,5 @@ bun.lock
 **/dist/
 **/build/
 **/_generated/
+
+packages/replicate/assets/

AGENTS.md

@@ -62,14 +62,14 @@ schema.prose(); // Validator for prose fields
 
 ```typescript
 collection.create(); // Main entry point - create lazy-initialized collections
-persistence.web.sqlite(); // Browser wa-sqlite Web Worker + OPFSCoopSyncVFS
+persistence.web.sqlite.create(); // Browser wa-sqlite Web Worker + IDBBatchAtomicVFS (IndexedDB)
 persistence.web.sqlite.once(); // SQLite singleton mode (shared across collections)
-persistence.web.encrypted(); // Browser encrypted storage (WebAuthn PRF)
-persistence.native.sqlite(); // React Native SQLite persistence (op-sqlite)
-persistence.native.encrypted(); // React Native encrypted storage (not yet implemented)
-persistence.memory(); // In-memory persistence (testing)
-persistence.custom(); // Custom storage adapter
-schema.prose(); // Zod schema for prose fields
+persistence.web.encryption.create(); // Browser encrypted storage (WebAuthn PRF)
+persistence.web.encryption.manager(); // Optional encryption lifecycle manager
+persistence.native.sqlite.create(); // React Native SQLite persistence (op-sqlite)
+persistence.native.encryption.create(); // React Native encrypted storage (not yet implemented)
+persistence.memory.create(); // In-memory persistence (testing)
+persistence.custom.create(); // Custom storage adapter
 schema.prose.extract(); // Extract plain text from prose JSON
 schema.prose.empty(); // Create empty prose value
 ```

CHANGELOG.md

@@ -5,6 +5,28 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- Manual connection gating, read-only collections, captured application request arguments, and awaitable connect/disconnect/reset lifecycle.
+- Versioned, checksummed self-hostable SQLite runtime assets and offline browser process-restart tests.
+- Bounded, resumable `migrateSnapshots` component migration for legacy shared snapshot sequence positions.
+
+### Fixed
+
+- Persisted rows become ready without waiting for network recovery.
+- Deletion and recovery absence drain/delete durable document state; failed writes cannot advance cursors or acknowledge undurable vectors.
+- Raw updates with missing Yjs dependencies survive restart; stale network generations cannot repopulate reset replicas.
+- Disconnect/reset followed by collection cleanup stops each Convex subscription once and ignores queued callbacks after unsubscribe.
+- Full-row updates remove omitted optional fields, SQLite document clearing is atomic, and browser worker shutdown drains accepted work while rejecting new work after closing starts.
+- Snapshot/delta discovery works after compaction, including scheduled compaction and writes between small pages.
+- Rejected deletes persist a marker-removal rollback before republishing and retain private full-state repair metadata across restart until the next explicit update or delete succeeds.
+
+### Upgrade notes
+
+Existing v2 component collections must complete snapshot migration before streaming. See [Durable offline replicas](docs/offline-replicas.md) for mixed-client support, pending writable creations, third-party persistence capabilities, runtime provenance, and native verification limits.
+
 ## [1.2.0-preview.0] - 2025-12-31
 
 ### Added

CLAUDE.md

@@ -19,7 +19,7 @@ Single package with exports:
 
 - `@trestleinc/replicate/client` - Client utilities (browser/React/Svelte)
 - `@trestleinc/replicate/server` - Server helpers (Convex functions)
-- `@trestleinc/replicate/shared` - Shared types (ProseFields, XmlFragmentJSON, OperationType)
+- `@trestleinc/replicate` - Shared types (ProseFields, XmlFragmentJSON, OperationType)
 - `@trestleinc/replicate/convex.config` - Component configuration
 
 ## Development Commands
@@ -53,8 +53,8 @@ src/
 │   ├── persistence/         # Swappable storage backends
 │   │   ├── types.ts         # Persistence, KeyValueStore interfaces
 │   │   ├── sqlite/          # SQLite backends
-│   │   │   ├── web.ts       # wa-sqlite Web Worker + OPFSCoopSyncVFS
-│   │   │   ├── worker.ts    # Web Worker with wa-sqlite (CDN loaded)
+│   │   │   ├── web.ts       # wa-sqlite Web Worker + IDBBatchAtomicVFS (IndexedDB)
+│   │   │   ├── worker.ts    # Web Worker with configurable wa-sqlite assets
 │   │   │   ├── schema.ts    # SQLite schema + persistence providers
 │   │   │   └── native.ts    # op-sqlite (React Native)
 │   │   ├── memory.ts        # Testing: in-memory
@@ -163,16 +163,16 @@ Server update (via stream subscription)
 collection.create(); // Create lazy-initialized collection (SSR-safe)
 
 // Persistence providers (namespaced by platform)
-persistence.web.sqlite(); // Browser: wa-sqlite Web Worker + OPFSCoopSyncVFS
+persistence.web.sqlite.create(); // Browser: wa-sqlite Web Worker + IDBBatchAtomicVFS (IndexedDB)
 persistence.web.sqlite.once(); // SQLite singleton (shared across collections)
-persistence.web.encrypted(); // Browser: encrypted storage with WebAuthn PRF
-persistence.native.sqlite(); // React Native: op-sqlite
-persistence.native.encrypted(); // React Native: encrypted storage (not yet implemented)
-persistence.memory(); // Testing: in-memory (cross-platform)
-persistence.custom(); // Custom storage adapter (cross-platform)
+persistence.web.encryption.create(); // Browser: encrypted storage with WebAuthn PRF
+persistence.web.encryption.manager(); // Optional encryption lifecycle manager
+persistence.native.sqlite.create(); // React Native: op-sqlite
+persistence.native.encryption.create(); // React Native: encrypted storage (not yet implemented)
+persistence.memory.create(); // Testing: in-memory (cross-platform)
+persistence.custom.create(); // Custom storage adapter (cross-platform)
 
 // Schema helpers (matches server API)
-schema.prose(); // Zod schema for prose fields
 schema.prose.extract(); // Extract plain text from ProseMirror JSON
 schema.prose.empty(); // Create empty prose value
 ```
@@ -187,7 +187,7 @@ schema.table(); // Define replicated table schema
 schema.prose(); // Validator for ProseMirror JSON
 ```
 
-### Shared (`@trestleinc/replicate/shared`)
+### Shared (`@trestleinc/replicate`)
 
 ```typescript
 ProseFields<T>; // Extract prose field names from document type
@@ -216,15 +216,19 @@ export const { stream, material, insert, update, remove, versions } = r<Task>({
 
 ```typescript
 import { collection, persistence } from '@trestleinc/replicate/client';
+import SqliteWorker from '@trestleinc/replicate/worker?worker';
 import { ConvexClient } from 'convex/browser';
 
+const sqlite = persistence.web.sqlite.once({
+	name: 'tasks',
+	assetBase: '/replicate-sqlite',
+	worker: () => new SqliteWorker(),
+});
+
 export const tasks = collection.create({
-	persistence: async () => {
-		const SQL = await initSqlJs({ locateFile: (f) => `/${f}` });
-		return persistence.sqlite.browser(SQL, 'tasks');
-	},
+	schema: taskSchema,
+	persistence: sqlite,
 	config: () => ({
-		schema: taskSchema,
 		convexClient: new ConvexClient(import.meta.env.VITE_CONVEX_URL),
 		api: api.tasks,
 		getKey: (task) => task.id,
@@ -233,10 +237,10 @@ export const tasks = collection.create({
 
 // Initialize once during app startup (browser only)
 await tasks.init();
-const collection = tasks.get();
+const rows = tasks.get();
 
 // Prose binding for rich text
-const binding = await collection.utils.prose(id, 'content');
+const binding = await rows.doc(id).prose('content');
 ```
 
 ### Sync Registration (Internal)

README.md

@@ -1,6 +1,8 @@
 # Replicate
 
-A local-first sync engine for [Convex](https://convex.dev). Replicate keeps a full copy of your data in client-side SQLite (via OPFS on web, op-sqlite on React Native) and syncs changes through Convex using CRDTs (Yjs). Users get instant reads, offline writes, and real-time collaboration without writing conflict resolution logic.
+A local-first sync engine for [Convex](https://convex.dev). Replicate keeps a full copy of your data in client-side SQLite (via IndexedDB-backed SQLite on web, op-sqlite on React Native) and syncs changes through Convex using CRDTs (Yjs). Users get instant reads, offline writes, and real-time collaboration without writing conflict resolution logic.
+
+For gated synchronization, read-only replicas, durable reset, browser precaching, and existing-component upgrades, see [Durable offline replicas](docs/offline-replicas.md). Existing v2 snapshots require the documented migration before streaming.
 
 ## Architecture
 
@@ -36,7 +38,7 @@ graph TB
         App[Application]
         Collection[TanStack DB Collection]
         Yjs[Yjs CRDT Documents]
-        SQLite[SQLite / OPFS]
+        SQLite[SQLite / IndexedDB]
     end
 
     subgraph Server
@@ -79,7 +81,7 @@ Install the Replicate component in your Convex project:
 ```typescript
 // convex/convex.config.ts
 import { defineApp } from 'convex/server';
-import replicate from '@trestleinc/replicate/component';
+import replicate from '@trestleinc/replicate/convex.config';
 
 const app = defineApp();
 app.use(replicate);
@@ -190,6 +192,7 @@ import type { Infer } from 'convex/values';
 
 const sqlite = persistence.web.sqlite.once({
 	name: 'myapp',
+	assetBase: '/replicate-sqlite',
 	worker: async () => {
 		const { default: SqliteWorker } = await import('@trestleinc/replicate/worker?worker');
 		return new SqliteWorker();
@@ -209,6 +212,9 @@ export const tasks = collection.create({
 export type Task = Infer<typeof taskSchema.shape>;
 ```
 
+For offline startup, copy the packaged runtime with `replicate-sqlite-assets public/replicate-sqlite`
+and precache it with the application shell. See [Durable offline replicas](docs/offline-replicas.md#browser-assets-and-offline-startup).
+
 ### Step 4: Initialize and Use
 
 Initialize the collection with optional server-seeded data, then query it like any TanStack DB collection:
@@ -360,19 +366,20 @@ const empty = schema.prose.empty();
 
 Replicate ships with several persistence backends:
 
-| Provider                                     | Platform     | Description                                    |
-| -------------------------------------------- | ------------ | ---------------------------------------------- |
-| `persistence.web.sqlite.create(options)`     | Browser      | wa-sqlite in a Web Worker with OPFS storage    |
-| `persistence.web.sqlite.once(options)`       | Browser      | Same as above, but returns a singleton factory |
-| `persistence.native.sqlite.create(db, name)` | React Native | op-sqlite integration                          |
-| `persistence.memory.create()`                | Any          | In-memory (for testing)                        |
-| `persistence.custom.create(adapter)`         | Any          | Bring your own `StorageAdapter`                |
+| Provider                                     | Platform     | Description                                      |
+| -------------------------------------------- | ------------ | ------------------------------------------------ |
+| `persistence.web.sqlite.create(options)`     | Browser      | wa-sqlite in a Web Worker with IndexedDB storage |
+| `persistence.web.sqlite.once(options)`       | Browser      | Same as above, but returns a singleton factory   |
+| `persistence.native.sqlite.create(db, name)` | React Native | op-sqlite integration                            |
+| `persistence.memory.create()`                | Any          | In-memory (for testing)                          |
+| `persistence.custom.create(adapter)`         | Any          | Bring your own `StorageAdapter`                  |
 
 **Web SQLite options:**
 
 ```typescript
 interface WebSqliteOptions {
-	name: string; // Database name in OPFS
+	name: string; // Database name in IndexedDB
+	assetBase?: string; // Base URL for the compatible wa-sqlite runtime graph
 	worker: Worker | (() => Worker | Promise<Worker>);
 }
 ```
@@ -637,37 +644,34 @@ await configure({
 
 ### Encrypted Storage
 
-Encrypt the local SQLite database using WebAuthn PRF (Passkey-derived keys):
+Encrypt persisted document payloads and application key-value payloads using WebAuthn PRF (Passkey-derived keys). Storage metadata, including key names and the document-ID discovery index, remains visible to the backing persistence provider:
 
 ```typescript
 import { persistence } from '@trestleinc/replicate/client';
 
 // Check if the browser supports PRF
 const supported = await persistence.web.encryption.webauthn.supported();
+if (!supported) throw new Error('WebAuthn PRF is required by this configuration');
 
-// Create an encryption manager (handles key lifecycle)
-const manager = persistence.web.encryption.manager({
-	rpId: 'example.com',
-	rpName: 'My App',
-	onStateChange: (state) => {
-		/* update UI */
+const storage = await persistence.web.sqlite.create({
+	name: 'myapp-encrypted',
+	assetBase: '/replicate-sqlite',
+	worker: async () => {
+		const { default: W } = await import('@trestleinc/replicate/worker?worker');
+		return new W();
 	},
 });
 
-// Create encrypted persistence
-const encrypted = () =>
-	persistence.web.encryption.create({
-		manager,
-		name: 'myapp-encrypted',
-		worker: async () => {
-			const { default: W } = await import('@trestleinc/replicate/worker?worker');
-			return new W();
-		},
-	});
+const encrypted = await persistence.web.encryption.create({
+	storage,
+	user: currentUser.id,
+	unlock: { webauthn: true },
+});
+await encrypted.unlock();
 
 const tasks = collection.create({
 	schema: taskSchema,
-	persistence: encrypted,
+	persistence: async () => encrypted,
 	config: () => ({
 		/* ... */
 	}),
@@ -835,6 +839,9 @@ function create<T extends object>(options: CreateCollectionOptions<T>): LazyColl
 interface CreateCollectionOptions<T> {
 	schema: VersionedSchema<GenericValidator>;
 	persistence: () => Promise<Persistence>;
+	networkMode?: 'auto' | 'manual'; // Defaults to auto
+	readOnly?: boolean; // Defaults to false
+	requestArgs?: () => Record<string, Value>; // Captured for each connection
 	config: () => {
 		convexClient: ConvexClient;
 		api: ConvexCollectionApi;
@@ -847,6 +854,10 @@ interface CreateCollectionOptions<T> {
 }
 ```
 
+`tasks.get()` returns the TanStack collection with `connect()`, `disconnect()`, and `reset()`
+methods. In manual mode, call and await `connect()` after credentials are available. Await
+`disconnect()` before changing account context and `reset()` when clearing the durable replica.
+
 #### `LazyCollection<T>`
 
 ```typescript
@@ -994,11 +1005,11 @@ interface UserIdentity {
 }
 ```
 
-### Shared Types (`@trestleinc/replicate/shared`)
+### Shared Types (`@trestleinc/replicate`)
 
 ```typescript
-import type { ProseValue, Duration, CompactionConfig } from '@trestleinc/replicate/shared';
-import { parseDuration } from '@trestleinc/replicate/shared';
+import type { ProseValue, Duration, CompactionConfig } from '@trestleinc/replicate';
+import { parseDuration } from '@trestleinc/replicate';
 
 // Duration strings
 type Duration = `${number}${'m' | 'h' | 'd'}`;
@@ -1064,7 +1075,7 @@ A full-featured offline-first issue tracker demonstrating real-world usage.
 
 **Features demonstrated:**
 
-- Offline-first with wa-sqlite persistence (SQLite in OPFS)
+- Offline-first with wa-sqlite persistence (SQLite in IndexedDB)
 - Rich text editing with TipTap + Yjs collaboration
 - PWA with custom service worker for offline navigation
 - Real-time sync across devices

apps/sveltekit/package.json

@@ -41,7 +41,6 @@
 		"convex": "catalog:",
 		"convex-svelte": "catalog:",
 		"tailwind-merge": "catalog:",
-		"wa-sqlite": "catalog:",
 		"lib0": "catalog:",
 		"y-protocols": "catalog:",
 		"yjs": "catalog:",