From 0101a8c0948f39f6bd3819371aeadd2ab0afce49 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 07:00:56 +0000 Subject: [PATCH 01/10] feat(driver-turso): remote face detects managed drift and applies index and column-drop entries The Turso REMOTE face refused detectManagedDrift and applyMigrationEntries, which left the v18 organization-ownership ceremony with no remote path. The detector now runs the shared differ over reads sent to the remote catalog (pragma_table_info, sqlite_master, pragma_index_list/info, the ADR-0120 D4 duplicate probe), against the declarations every remote schema door and the DDL-free registerObjectMetadata record. The apply door runs drop_index, create_index, recreate_index, replace_unique_index and drop_column as one libSQL write batch each, verified by read-back; other column entries need the SQLite rebuild and are skipped and logged. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01Lr3rg6qTp9r38exA5DHEEH --- .../driver-turso/src/remote-transport.ts | 73 ++- .../drivers/driver-turso/src/turso-driver.ts | 521 ++++++++++++++---- 2 files changed, 469 insertions(+), 125 deletions(-) diff --git a/packages/drivers/driver-turso/src/remote-transport.ts b/packages/drivers/driver-turso/src/remote-transport.ts index 09f9430b86f..ed36ba9dc67 100644 --- a/packages/drivers/driver-turso/src/remote-transport.ts +++ b/packages/drivers/driver-turso/src/remote-transport.ts @@ -1307,6 +1307,21 @@ export class RemoteTransport { return result.rows; } + /** + * [#23816] Run statements as one libSQL batch and answer each one's rows. + * + * A `'write'` batch runs inside one transaction on the server: a statement + * that fails rolls back every statement before it, so the batch lands whole + * or not at all. That is what the remote migration doors rely on to drop an + * index and the column it keys, or to drop and re-create an index under one + * name, without ever leaving the half-applied state between the two. + */ + async batch(statements: InStatement[], mode: 'read' | 'write'): Promise> { + const client = await this.ensureConnected(); + const results = await client.batch(statements, mode); + return results.map((result) => result.rows); + } + // =================================== // CRUD Operations // =================================== @@ -2488,38 +2503,52 @@ export class RemoteTransport { continue; } if (emitted.has(index.name)) continue; - // The KEY columns are references and stay gated. The index NAME is not a - // reference but one name by definition — the class #14113 escapes for an - // alias rather than refuses. It is the one position here an author types - // freely (`IndexSchema.name` is any string, and the local face quotes - // whatever it is given), so refusing it would fail this face's WHOLE - // schema sync over a declaration the other face accepts. - for (const column of index.columns) this.assertSafeIdentifier(column); - const nameSql = this.aliasIdentifierSql(index.name); - - const nullSafe = new Set(index.nullSafeColumns ?? []); - const parts = index.columns.map((c) => - nullSafe.has(c) ? organizationKeyPartSql(`"${c}"`) : `"${c}"`, - ); - // `IF NOT EXISTS` is what makes a concurrent creator (two instances - // booting against one database) a no-op rather than an error — and it is - // also what makes the per-statement retry in - // {@link retrofitDeclaredIndexes} safe under EITHER libsql batch semantic - // (transactional: nothing was applied; non-transactional: re-applying is - // a no-op). planned.push({ name: index.name, table: tableName, unique: index.unique, - sql: - `CREATE ${index.unique ? 'UNIQUE ' : ''}INDEX IF NOT EXISTS ${nameSql} ` + - `ON "${tableName}" (${parts.join(', ')})`, + sql: this.createIndexSql(tableName, index), }); emitted.add(index.name); } return planned; } + /** + * The `CREATE INDEX` this face issues for one index: the schema sync's + * declared indexes ({@link buildDeclaredIndexDDL}) and, since #23816, the + * `create_index` / `recreate_index` drift entries `TursoDriver` applies on + * the remote face. One emitter for both, so an index the migration door + * re-creates is byte-for-byte the one the sync would have created, and the + * drift differ reads the two alike. + */ + createIndexSql( + tableName: string, + index: { name: string; columns: readonly string[]; unique: boolean; nullSafeColumns?: readonly string[] }, + ): string { + // The table and KEY columns are references and stay gated. The index NAME + // is not a reference but one name by definition — the class #14113 escapes + // for an alias rather than refuses. It is the one position here an author + // types freely (`IndexSchema.name` is any string, and the local face quotes + // whatever it is given), so refusing it would fail this face's WHOLE schema + // sync over a declaration the other face accepts. + this.assertSafeIdentifier(tableName); + for (const column of index.columns) this.assertSafeIdentifier(column); + const nameSql = this.aliasIdentifierSql(index.name); + + const nullSafe = new Set(index.nullSafeColumns ?? []); + const parts = index.columns.map((c) => + nullSafe.has(c) ? organizationKeyPartSql(`"${c}"`) : `"${c}"`, + ); + // `IF NOT EXISTS` is what makes a concurrent creator (two instances + // booting against one database) a no-op rather than an error — and it is + // also what makes the per-statement retry in + // {@link retrofitDeclaredIndexes} safe under EITHER libsql batch semantic + // (transactional: nothing was applied; non-transactional: re-applying is + // a no-op). + return `CREATE ${index.unique ? 'UNIQUE ' : ''}INDEX IF NOT EXISTS ${nameSql} ON "${tableName}" (${parts.join(', ')})`; + } + /** * [#17609] The planned indexes the database does not already carry, by name — * the remote twin of the `existing.has(name)` skip in diff --git a/packages/drivers/driver-turso/src/turso-driver.ts b/packages/drivers/driver-turso/src/turso-driver.ts index d49e052c151..826c0160c90 100644 --- a/packages/drivers/driver-turso/src/turso-driver.ts +++ b/packages/drivers/driver-turso/src/turso-driver.ts @@ -31,12 +31,18 @@ import { SqlDriver, GLOBAL_TENANT, + applyIndexKeyParts, + isIndexDriftOp, + parseIndexDdl, + type IntrospectedColumn, type IntrospectedSchema, type ManagedDriftEntry, + type PhysicalIndex, type SqlDriverConfig, type SqlWindowFunctionQuery, } from '@objectstack/driver-sql'; import { StandardErrorCode } from '@objectstack/spec/api'; +import { StorageNameMapping } from '@objectstack/spec/system'; import type { DriverQuery } from '@objectstack/spec/contracts'; import type { DriverOptions, FilterCondition } from '@objectstack/spec/data'; import type { Client } from '@libsql/client'; @@ -538,79 +544,82 @@ function refuseRemoteDeferredDdl(): never { throw err; } -// ── Remote schema drift detection: refused, never "no drift" ───────────────── +// ── Remote schema drift: read through the libSQL client ───────────────────── /** - * [#19845] The Turso REMOTE face cannot detect schema drift, and now says so - * instead of answering that there is none. + * [#23816] The Turso REMOTE face detects managed-schema drift and applies the + * index and column-drop entries of a drift report, through the libSQL client. * - * # The defect this replaces + * # What it replaces * - * `SqlDriver.detectManagedDrift` reads the physical schema through Knex: a - * `hasTable` probe per table, then column and index introspection fed to the - * shared differ. In remote mode that Knex instance was then the placeholder - * `:memory:` database {@link TursoDriver.toKnexConfig} handed the base - * constructor. It held none of this datasource's tables, so every table was - * skipped as absent and the answer was `[]` whatever the remote database held. - * The no-argument call had a second reason to answer `[]`: it iterates - * `managedObjectFields`, which only the Knex `initObjects` fills and no remote - * schema door reaches. [#20054] Remote mode's Knex has no connection at all - * now, so an explicit-objects call would fail on its first `hasTable`; the - * no-argument call would still answer `[]` from the empty registry, which is - * why the refusal stays. Measured on the transport's SQLite-backed double - * (`turso-remote-drift-detection-refusal.test.ts`): a synced table carrying an - * extra physical column the declaration omits reads `unmapped_column` / - * `drop_column` on the local face and `[]` on the remote one, with or without - * explicit objects. The artifact-pinned boot gate of `os serve`, whose job is - * to refuse a boot on destructive drift, therefore let every remote-Turso boot - * through as never drifted. + * Both members refused here with `NOT_IMPLEMENTED` / 501 (#19845, #20055). The + * refusal replaced a worse answer: the inherited detector read the physical + * schema through a Knex instance remote mode builds with no connection, and + * walked a registry no remote schema door filled, so it answered "no drift" + * for every remote database. The refusal was honest, and it left the v18 + * organization-ownership ceremony (`os migrate organization-ownership`) with + * no remote path at all: its column drops go through + * `detectManagedDrift` + `applyMigrationEntries`, so every hosted tenant on + * this transport stopped at its first v18 request. * - * # Why a refusal rather than an implementation + * # How the detector answers here * - * The shared differ would serve a remote table: a clean remote-synced table, - * judged through a local Knex connection to the same SQLite file, reports no - * entries, as the local face does. But every read that feeds the differ goes - * through `this.knex` (table existence, column facts and order, the index set, - * the NULL-safe duplicate probe), so a remote implementation is a second copy - * of each of those SQLite arms. It also needs a remote answer for - * `applyMigrationEntries`, which the gate calls on whatever it finds and which - * runs through the same `this.knex`. Until that exists the refusal is the honest - * answer, in the envelope and for the reason {@link refuseRemoteDeferredDdl} - * records for its sibling gap on this transport: the call is spelled correctly - * and the base class declares it, so the gap is the backend's. - * `NOT_IMPLEMENTED`/501 is a {@link StandardErrorCode} member, so there is no - * new code. + * The verdict is still the shared differ's (`SqlDriver.detectTableDrift`, the + * same `diffManagedTable` / `diffManagedIndexes` the local face runs). Only + * the READS that feed it are this face's own, and each sends the statement the + * local SQLite arm sends, to the remote database: * - * # What a caller sees + * - the columns: `pragma_table_info`, shaped as Knex's SQLite `columnInfo()` + * shapes it ({@link remoteIntrospectedColumn}); + * - the indexes: `sqlite_master` and `pragma_index_list` / `pragma_index_info`, + * parsed by the driver-sql parsers the local arm uses; + * - the NULL-safe duplicate probe the ADR-0120 D4 pre-flight runs; + * - and table existence, from `sqlite_master`. * - * The boot gate already has a channel for "the check did not run": a throw - * from `detectManagedDrift` becomes a warning carrying this message, and the - * boot continues. That is the right reading of a driver that cannot judge. It - * is neither a drift verdict that would refuse every remote boot nor a - * silence. The `os migrate` commands that read drift never get this far on a - * remote datasource, because they arm deferred DDL first and that is refused. + * The declared side is {@link TursoDriver.remoteDeclaredSchemas}: what each + * object that reached this face declares, recorded by every remote schema door + * and by the DDL-free registration a `skipSchemaSync` boot takes. + * `turso-local-remote-drift-parity.test.ts` holds the two faces to one answer + * over one SQLite file, both for a clean table and for each drifted shape. + * + * # What the apply door answers here + * + * The entries that are DDL this transport can send as it stands: `drop_index`, + * `create_index`, `recreate_index` and `replace_unique_index` (portable index + * DDL on every dialect), and `drop_column` (`ALTER TABLE … DROP COLUMN`, which + * SQLite and libSQL support since 3.35). A column's indexes go with it, in the + * same transaction, as they do on PostgreSQL and in the local face's SQLite + * table rebuild. Every other column entry (nullability, width, a dropped + * DEFAULT) needs that rebuild, which this transport has no copy of, so it is + * reported `skipped` and logged, as the in-place arm reports an op its dialect + * cannot perform. Each entry is reported `applied` only once a read-back shows + * the physical schema in its target state. */ -function refuseRemoteDriftDetection(): never { - const err = new Error( - 'Schema drift detection is not supported by the Turso REMOTE transport (this datasource\'s ' + - 'transport mode is `remote`), so this driver cannot say whether the database\'s physical ' + - 'schema matches the declared objects. Drift detection reads the physical schema through the ' + - 'SQL driver\'s Knex connection, which remote mode does not have: every statement goes to the ' + - 'remote database through the libSQL client instead. The objects it compares by default are the ' + - 'ones the SQL driver\'s own schema sync registers, and remote mode registers none there, so ' + - 'answering would report "no drift" for every remote database, whatever its tables hold; the ' + - 'call refuses instead. The call is spelled ' + - 'correctly and `SqlDriver` declares it, so this is a capability gap of the remote transport ' + - 'rather than a mistake in the request, which is why it answers NOT_IMPLEMENTED/501 and not a ' + - '400. To check this database for drift, run `os migrate plan` against a local SQLite copy of it ' + - '(a `file:` URL), where the physical schema is introspected. Pointed at the remote URL, ' + - '`os migrate plan` refuses, because the remote transport cannot defer schema DDL.', - ) as Error & { code?: string; status?: number }; - err.code = StandardErrorCode.enum.NOT_IMPLEMENTED; - err.status = 501; - throw err; + +/** + * One `pragma_table_info` row as Knex's SQLite `columnInfo()` reports it — the + * shape `SqlDriver.introspectColumns` reads its per-column facts from: the + * declared type lower-cased with any `(n)` split off into `maxLength` (a + * string, as Knex leaves it), `nullable` from `notnull`, and the raw DEFAULT. + */ +function remoteIntrospectedColumn(row: Record): IntrospectedColumn { + const declared = String(row.type ?? ''); + const width = declared.match(/.*\((\d+)\)/); + const type = (width ? declared.split('(')[0]! : declared).toLowerCase(); + return { + name: String(row.name), + type: type || 'string', + nullable: !Number(row.notnull ?? 0), + defaultValue: row.dflt_value as IntrospectedColumn['defaultValue'], + primaryKey: false, + isUnique: false, + ...(width ? { maxLength: width[1] } : {}), + } as IntrospectedColumn; } +/** A SQLite identifier, quoted, with any embedded quote doubled. */ +const quoteRemoteIdentifier = (name: string): string => `"${String(name).replace(/"/g, '""')}"`; + // ── Remote media column move planning: refused, never "nothing to move" ────── /** @@ -641,10 +650,10 @@ function refuseRemoteDriftDetection(): never { * * # Why a refusal rather than an implementation * - * Planning remotely needs a remote copy of the SQLite arms the planner reads - * through Knex (table existence and column introspection), which is the same - * second copy {@link refuseRemoteDriftDetection} declines for drift. It would - * also need a remote answer to the step after the plan: the command stamps + * Planning remotely needs remote answers to the SQLite reads the planner makes + * through Knex (table existence and column introspection). [#23816] The remote + * drift detector now has those reads (see "Remote schema drift" above), but the + * planner would also need a remote answer to the step after the plan: the command stamps * `columns_moved_at` so the driver writes bare ids from its next boot, and the * remote face never asks for that stamp. The ADR-0104 resolver is supplied to * this driver, but only `SqlDriver.initObjects` asks it, and none of the three @@ -653,7 +662,7 @@ function refuseRemoteDriftDetection(): never { * times on every remote door, and a remote write stored `"file_abc"` (the JSON * encoding) and read it back as `file_abc`. So until both halves exist the * refusal is the honest answer, in the envelope and for the reason - * {@link refuseRemoteDriftDetection} records for its sibling gap on this + * {@link refuseRemoteDeferredDdl} records for its sibling gap on this * transport: the call is spelled correctly and the base class declares it, so * the gap is the backend's. `NOT_IMPLEMENTED`/501 is a * {@link StandardErrorCode} member, so there is no new code. @@ -690,7 +699,7 @@ function refuseRemoteMediaColumnMove(): never { /** * [#20055] The refusal for a `SqlDriver` member the REMOTE face cannot answer, - * in the envelope and the shape of {@link refuseRemoteDriftDetection} and + * in the envelope and the shape of {@link refuseRemoteDeferredDdl} and * {@link refuseRemoteMediaColumnMove}: the operation, what the caller cannot * have, why this face cannot give it, the 501 rationale, and where the answer * does exist. @@ -921,7 +930,9 @@ export const REMOTE_FACE_ANSWERS = { // the table it records (`remoteTableFor`), as `getBuilder` does locally. registerExternalObject: 'inherited', // In-memory bookkeeping with no Knex, by its own contract (`skipSchemaSync`). - registerObjectMetadata: 'inherited', + // [#23816] Redeclared: the remote face also records what each object + // declares, the input its drift detector diffs against. + registerObjectMetadata: 'remote', // Deferred DDL: arming is refused, so nothing is ever deferred here, and the // three inherited members answer that truthfully without reaching Knex. setDeferredDdl: 'refused', @@ -931,9 +942,11 @@ export const REMOTE_FACE_ANSWERS = { // `{ created: 0, existing: 0 }`: the remote doors count nothing, and both // counts at zero is the `IDataDriver` contract's "cannot say". getSchemaSyncStats: 'inherited', - // Drift and migration: every read and write goes through Knex. - detectManagedDrift: 'refused', - applyMigrationEntries: 'refused', + // [#23816] Drift and migration: the shared differ over the remote catalog, + // and the index / column-drop entries applied through the libSQL client (see + // "Remote schema drift" above for which entries, and why only those). + detectManagedDrift: 'remote', + applyMigrationEntries: 'remote', planMediaColumnMove: 'refused', // `false`, the method's own "not taken" answer: only the Knex `initObjects` // asks the resolver, so on this face it would never be asked. @@ -1698,6 +1711,36 @@ export class TursoDriver extends SqlDriver { */ private readonly remoteManagedObjects = new Set(); + /** + * [#23816] What each object that reached this REMOTE face declares, keyed by + * its table: the field set and `indexes` the remote drift detector + * ({@link detectManagedDrift}) diffs the physical schema against. + * + * The local faces keep the same fact in `managedObjectFields` / + * `managedObjectIndexes`, which `SqlDriver.registerObjectMetadata` fills on + * the way into the Knex `initObjects`. The remote schema doors never reach + * that method, so this face records it at the two places an object reaches + * it: every remote schema door ({@link registerRemoteFieldMetadata}) and the + * DDL-free {@link registerObjectMetadata} a `skipSchemaSync` boot takes. It is + * kept apart from `managedObjectFields` on purpose: that map answers other + * questions on this class too (the cross-field column set, the pagination + * tie-breaker), and filling it from the schema doors would change those + * remote answers as well. + */ + private readonly remoteDeclaredSchemas = new Map; indexes?: any[] }>(); + + /** + * Record one object in {@link remoteDeclaredSchemas}. Overwritten on every + * registration, as the local registry is: a re-registration that drops + * `indexes` must stop the detector expecting them. + */ + private recordRemoteDeclaredSchema(obj: { name: string; fields?: Record; indexes?: unknown }): void { + this.remoteDeclaredSchemas.set(StorageNameMapping.resolveTableName(obj), { + fields: obj.fields ?? {}, + ...(Array.isArray(obj.indexes) ? { indexes: obj.indexes } : {}), + }); + } + /** * [#19868] Remote `date` / json columns whose storage backfill found nothing * left to do in THIS process, so later schema syncs skip them. In memory @@ -3225,8 +3268,9 @@ export class TursoDriver extends SqlDriver { * `id` primary key whether or not the best-effort coercion registration below * does. */ - private registerRemoteFieldMetadata(obj: { name: string; fields?: Record; tenancy?: any }): void { + private registerRemoteFieldMetadata(obj: { name: string; fields?: Record; tenancy?: any; indexes?: unknown }): void { this.remoteManagedObjects.add(obj.name); + this.recordRemoteDeclaredSchema(obj); try { this.registerExternalObject({ name: obj.name, fields: obj.fields, tenancy: obj.tenancy }); } catch { @@ -3261,7 +3305,7 @@ export class TursoDriver extends SqlDriver { * steady state at one round-trip per backfill. */ private async completeRemoteSchemaSync( - objects: Array<{ name: string; fields?: Record; tenancy?: any }>, + objects: Array<{ name: string; fields?: Record; tenancy?: any; indexes?: unknown }>, ): Promise { if (objects.length === 0) return; for (const obj of objects) this.registerRemoteFieldMetadata(obj); @@ -3668,13 +3712,13 @@ export class TursoDriver extends SqlDriver { } /** - * Detect managed-schema drift — refused on the REMOTE face, see - * {@link refuseRemoteDriftDetection}. The inherited detector reads the - * physical schema through `this.knex`, which remote mode builds with no - * connection; with no `objects` it walks a registry no remote schema door - * fills and answers `[]`. Refused with or without explicit `objects`, because - * neither can judge the remote database. Local and replica modes inherit the - * Knex detector unchanged. + * Detect managed-schema drift. [#23816] On the REMOTE face: the shared + * differ, fed by the remote catalog (see "Remote schema drift" at the top of + * this file). With no `objects` it walks {@link remoteDeclaredSchemas}, which + * every remote schema door and the DDL-free {@link registerObjectMetadata} + * fill, as the local face walks `managedObjectFields`. A table the database + * does not hold is skipped, as the local face skips it. Local and replica + * modes inherit the Knex detector unchanged. * * The parameter repeats the base's declared shape key for key rather than * deriving it (`check:object-def-param-keys` arm C), so the keys a caller may @@ -3683,8 +3727,130 @@ export class TursoDriver extends SqlDriver { override async detectManagedDrift( objects?: Array<{ name: string; fields?: Record; indexes?: any[] }>, ): ReturnType { - if (this.isRemote) refuseRemoteDriftDetection(); - return super.detectManagedDrift(objects); + if (!this.isRemote) return super.detectManagedDrift(objects); + const tables = new Map; indexes?: any[] }>(); + if (objects) { + for (const o of objects) { + tables.set(StorageNameMapping.resolveTableName(o), { fields: o.fields ?? {}, indexes: o.indexes }); + } + } else { + for (const [table, declared] of this.remoteDeclaredSchemas) tables.set(table, declared); + } + const present = new Set( + (await this.remoteCatalogRead(`SELECT name FROM sqlite_master WHERE type = 'table'`)).map((r) => String(r.name)), + ); + const out: ManagedDriftEntry[] = []; + for (const [table, declared] of tables) { + if (!present.has(table)) continue; + out.push(...(await this.detectTableDrift(table, declared.fields, declared.indexes))); + } + // The base detector's order: by table, then by column. + out.sort((a, b) => (a.table === b.table ? (a.column ?? '').localeCompare(b.column ?? '') : a.table.localeCompare(b.table))); + return out; + } + + /** + * [#23816] One catalog read on the REMOTE database, in the raw-statement + * fault envelope `execute()` answers with. + */ + private async remoteCatalogRead(sql: string, args: unknown[] = []): Promise>> { + let rows: unknown; + try { + rows = await this.remoteTransport!.execute(sql, args); + } catch (error) { + throw this.rawStatementFault(sql, error); + } + return (Array.isArray(rows) ? rows : []) as Array>; + } + + /** + * The table's columns, in declared order. [#23816] On the REMOTE face, read + * from `pragma_table_info` and shaped as Knex's SQLite `columnInfo()` shapes + * it ({@link remoteIntrospectedColumn}); the cid is the ordinal, as + * `introspectColumnOrder` reads it locally. + */ + protected override async introspectColumns(tableName: string): Promise { + if (!this.isRemote) return super.introspectColumns(tableName); + const rows = await this.remoteCatalogRead( + 'SELECT cid, name, type, "notnull", dflt_value FROM pragma_table_info(?) ORDER BY cid', + [tableName], + ); + return rows.map(remoteIntrospectedColumn); + } + + /** + * The table's indexes. [#23816] On the REMOTE face, the statements the local + * SQLite arm sends, sent to the remote database: `sqlite_master` for each + * index's DDL (the only place an expression key or a `WHERE` predicate + * survives), `pragma_index_list` for the set, and `pragma_index_info` for an + * index whose DDL does not parse. The DDL goes through the driver-sql + * parsers the local arm uses, so an index reads alike on both faces. + */ + protected override async introspectIndexes( + tableName: string, + opts: { onFailure?: 'throw' | 'partial' } = {}, + ): Promise { + if (!this.isRemote) return super.introspectIndexes(tableName, opts); + const byName = new Map(); + const upsert = (name: string, unique: boolean, primary: boolean): PhysicalIndex => { + let entry = byName.get(name); + if (!entry) byName.set(name, (entry = { name, columns: [], unique, primary })); + return entry; + }; + try { + const ddlByName = new Map(); + for (const r of await this.remoteCatalogRead( + `SELECT name, sql FROM sqlite_master WHERE type = 'index' AND tbl_name = ?`, + [tableName], + )) { + if (typeof r.sql === 'string' && r.sql) ddlByName.set(String(r.name), r.sql); + } + for (const r of await this.remoteCatalogRead( + 'SELECT name, "unique", origin, partial FROM pragma_index_list(?) ORDER BY seq', + [tableName], + )) { + const name = String(r.name); + const entry = upsert(name, Number(r.unique) === 1, r.origin === 'pk'); + const parsed = parseIndexDdl(ddlByName.get(name) ?? ''); + if (parsed) { + applyIndexKeyParts(entry, parsed.keyParts); + if (parsed.partial) entry.partial = true; + } else { + for (const c of await this.remoteCatalogRead('SELECT name FROM pragma_index_info(?) ORDER BY seqno', [name])) { + if (c.name != null) entry.columns.push(String(c.name)); + } + } + if (Number(r.partial) === 1) entry.partial = true; + } + } catch (e) { + // Only a caller that can CORRECT a short read may ask for one (#7332). + if (opts.onFailure !== 'partial') throw e; + } + return [...byName.values()]; + } + + /** + * The ADR-0120 D4 duplicate pre-flight. [#23816] On the REMOTE face, the + * statement the local face builds, with SQLite's own identifier quoting, + * sent to the remote database. + */ + protected override async probeNullSafeUniqueDuplicates( + table: string, + columns: string[], + nullSafeColumns: string[], + ): Promise> { + if (!this.isRemote) return super.probeNullSafeUniqueDuplicates(table, columns, nullSafeColumns); + const nullSafe = new Set(nullSafeColumns); + const parts = columns.map((c) => + nullSafe.has(c) ? `COALESCE(${quoteRemoteIdentifier(c)}, '${GLOBAL_TENANT}')` : quoteRemoteIdentifier(c), + ); + const sql = + `SELECT ${parts.map((p, i) => `${p} AS k${i}`).join(', ')}, COUNT(*) AS n FROM ${quoteRemoteIdentifier(table)} ` + + `GROUP BY ${parts.join(', ')} HAVING COUNT(*) > 1 ORDER BY n DESC LIMIT 20`; + return (await this.remoteCatalogRead(sql)).map((r) => ({ + key: columns.map((c, i) => `${c}=${JSON.stringify(r[`k${i}`])}`).join(', '), + rows: Number(r.n ?? 0), + })); } /** @@ -3795,6 +3961,21 @@ export class TursoDriver extends SqlDriver { return super.dropTable(object, options); } + /** + * The DDL-free registration a `skipSchemaSync` boot takes — inherited + * unchanged, and [#23816] on the REMOTE face it also records each object in + * {@link remoteDeclaredSchemas}, so a boot that ran no schema door still + * gives the drift detector the declarations to diff against. Still no + * statement and no round-trip. The parameter repeats the base's declared + * shape key for key (`check:object-def-param-keys` arm C). + */ + override registerObjectMetadata( + objects: Array<{ name: string; fields?: Record; tenancy?: any; indexes?: any[] }>, + ): void { + super.registerObjectMetadata(objects); + if (this.isRemote) for (const obj of objects) this.recordRemoteDeclaredSchema(obj); + } + // =================================== // Inherited SqlDriver members (remote mode overrides) // =================================== @@ -3941,9 +4122,10 @@ export class TursoDriver extends SqlDriver { /** * Schema introspection — refused on the REMOTE face. The inherited member * reads every table's columns, foreign keys, primary keys and unique - * constraints through Knex (`columnInfo()` and the SQLite pragmas), so a - * remote answer would be a second copy of those four readers, the copy - * {@link refuseRemoteDriftDetection} declines for drift. Its callers already + * constraints through Knex (`columnInfo()` and the SQLite pragmas). The + * remote drift detector (#23816) reads two of those four (the columns and + * the indexes) and no caller of this member has asked for the other two on + * this face, so the member is still refused. Its callers already * treat a throw as "could not introspect": the datasource connection test * answers `ok: false` with this message, and the federation validation sweep * rows the datasource `unreachable`. @@ -4067,30 +4249,163 @@ export class TursoDriver extends SqlDriver { } /** - * Apply drift entries — refused on the REMOTE face. Every entry is applied - * through Knex. The inherited member resolved here, reporting a destructive - * entry its caller had allowed as `skipped` (its SQLite rebuild caught the - * connection failure), so the answer read like a policy decision. The entries - * come from {@link detectManagedDrift}, which this face refuses too. The - * option parameter repeats the base's declared shape (`check:object-def-param-keys`). + * Apply drift entries. [#23816] On the REMOTE face, through the libSQL + * client: the index entries and `drop_column`, each in one `write` batch so + * it lands whole or not at all, and each reported `applied` only once a + * read-back shows its target state (see "Remote schema drift" at the top of + * this file for which entries, and why only those). The contract is the + * base's: a destructive entry is skipped unless `allowDestructive`, column + * entries run before index entries, and an entry this transport cannot + * perform, or whose DDL fails, is reported `skipped` and logged. The option + * parameter repeats the base's declared shape (`check:object-def-param-keys`). */ override async applyMigrationEntries( entries: ManagedDriftEntry[], opts: { allowDestructive?: boolean } = {}, ): ReturnType { - if (this.isRemote) { - refuseRemoteInheritedMember( - 'Applying schema migration entries (`applyMigrationEntries()`)', - 'this driver cannot change the physical schema of the remote database from a drift report.', - `The entries are applied through the SQL driver's Knex connection. ${REMOTE_HAS_NO_KNEX_CONNECTION} ` + - 'Until this change the call answered as if it had run, with every entry reported ' + - '`skipped`, including a destructive entry that `allowDestructive` permitted.', - 'An ordinary boot against this datasource (`os serve` / `os start`) performs the additive ' + - 'schema sync directly. A change that needs a table rebuild or a dropped column has no ' + - 'remote path in this driver.', + if (!this.isRemote) return super.applyMigrationEntries(entries, opts); + this.assertSchemaMutable('reconcileManagedSchema'); + const allowDestructive = opts.allowDestructive === true; + const applied: ManagedDriftEntry[] = []; + const skipped: ManagedDriftEntry[] = []; + const candidates = entries.filter((d) => { + if (d.category === 'destructive' && !allowDestructive) { + skipped.push(d); + return false; + } + return true; + }); + // Column entries FIRST, as the base runs them: a dropped column takes its + // indexes with it, which may already satisfy an index entry below. + const ordered = [ + ...candidates.filter((d) => !isIndexDriftOp(d.op)), + ...candidates.filter((d) => isIndexDriftOp(d.op)), + ]; + for (const d of ordered) { + try { + const ok = isIndexDriftOp(d.op) ? await this.applyRemoteIndexDriftOp(d.op) : await this.applyRemoteColumnDriftOp(d.op); + (ok ? applied : skipped).push(d); + } catch (e: any) { + this.logger.warn(`[schema-drift] failed to reconcile '${d.table}' on the Turso REMOTE transport`, e?.message ?? e); + skipped.push(d); + } + } + return { applied, skipped }; + } + + /** + * [#23816] One column drift entry on the REMOTE face. `drop_column` only: + * `ALTER TABLE … DROP COLUMN`, preceded in the same transaction by a + * `DROP INDEX` of every index keyed on the column, since SQLite refuses to + * drop a column an index still uses. That is the outcome the other dialects + * reach by themselves (PostgreSQL drops the column's indexes, the local + * SQLite rebuild re-creates only the declared ones). A column SQLite still + * refuses to drop (a primary key, a UNIQUE or foreign-key constraint, a + * view or trigger that names it) fails the batch, which rolls the dropped + * indexes back with it. Every other column entry needs the SQLite table + * rebuild, which this transport does not run, so it is skipped. + */ + private async applyRemoteColumnDriftOp(op: ManagedDriftEntry['op']): Promise { + const column = (op as { column?: string }).column; + if (op.type !== 'drop_column' || !column) { + this.logger.warn( + `[schema-drift] ${op.type} on ${op.table}.${column ?? ''} is unsupported on the Turso REMOTE transport — ` + + 'skipped: it needs the SQLite table rebuild, which this transport does not run', ); + return false; } - return super.applyMigrationEntries(entries, opts); + const hasColumn = async () => (await this.introspectColumns(op.table)).some((c) => c.name === column); + if (!(await hasColumn())) return false; + const keyed = (await this.introspectIndexes(op.table)).filter( + (i) => !i.primary && !i.name.startsWith('sqlite_autoindex_') && i.columns.includes(column), + ); + await this.remoteTransport!.batch( + [ + ...keyed.map((i) => `DROP INDEX IF EXISTS ${quoteRemoteIdentifier(i.name)}`), + `ALTER TABLE ${quoteRemoteIdentifier(op.table)} DROP COLUMN ${quoteRemoteIdentifier(column)}`, + ], + 'write', + ); + return !(await hasColumn()); + } + + /** + * [#23816] One index drift entry on the REMOTE face, mirroring + * `SqlDriver.applyIndexDriftOp` op for op. A drop-and-create pair runs as + * one `write` batch, so a create that fails (duplicates under a new UNIQUE, + * a column that is not there) leaves the old index exactly as it was. The + * `CREATE INDEX` is the schema sync's own ({@link RemoteTransport.createIndexSql}). + */ + private async applyRemoteIndexDriftOp(op: ManagedDriftEntry['op']): Promise { + const names = async () => new Set((await this.introspectIndexes(op.table)).map((i) => i.name)); + const create = (name: string, columns: string[], unique: boolean, nullSafeColumns?: string[]) => + this.remoteTransport!.createIndexSql(op.table, { name, columns, unique, ...(nullSafeColumns ? { nullSafeColumns } : {}) }); + const drop = (name: string) => `DROP INDEX IF EXISTS ${quoteRemoteIdentifier(name)}`; + switch (op.type) { + case 'drop_index': + // Applied once the name is gone, whether or not it was there: a re-run + // after a half-applied migration must find the index's step done. + await this.remoteTransport!.batch([drop(op.indexName)], 'write'); + return !(await names()).has(op.indexName); + case 'create_index': { + if (!(await this.remoteIndexColumnsPresent(op.table, op.indexName, op.columns))) return false; + await this.remoteTransport!.batch([create(op.indexName, op.columns, op.unique, op.nullSafeColumns)], 'write'); + return (await names()).has(op.indexName); + } + case 'recreate_index': { + // ADR-0120 D4: the NULL-safe tightening re-runs the duplicate pre-flight + // immediately before the drop, as the local face does. + if (op.unique && (op.nullSafeColumns?.length ?? 0) > 0) { + const duplicates = await this.probeNullSafeUniqueDuplicates(op.table, op.columns, op.nullSafeColumns!); + if (duplicates.length > 0) { + this.logDurabilityFailure( + `[schema-drift] REFUSING to rebuild '${op.indexName}' on '${op.table}' as a NULL-safe unique — ` + + `${duplicates.length} duplicate group(s) violate it (e.g. ${duplicates[0].key} × ${duplicates[0].rows} rows). ` + + `The existing index is left in place; deduplicate and re-run "os migrate plan" (ADR-0120 D4).`, + ); + return false; + } + } + if (!(await this.remoteIndexColumnsPresent(op.table, op.indexName, op.columns))) return false; + await this.remoteTransport!.batch( + [drop(op.indexName), create(op.indexName, op.columns, op.unique, op.nullSafeColumns)], + 'write', + ); + return (await names()).has(op.indexName); + } + case 'replace_unique_index': { + // CREATE before DROP, in one transaction: uniqueness is never + // unenforced in between, and a create that fails drops nothing. + if (!(await this.remoteIndexColumnsPresent(op.table, op.createIndexName, op.createColumns))) return false; + await this.remoteTransport!.batch( + [create(op.createIndexName, op.createColumns, true, op.nullSafeColumns), ...op.dropIndexNames.map(drop)], + 'write', + ); + const after = await names(); + return after.has(op.createIndexName) && op.dropIndexNames.every((n) => !after.has(n)); + } + default: + // `unbuildable_index` is report-only: no column to build it over. + return false; + } + } + + /** + * [#23816] Whether every key column of an index the remote apply door would + * create is a physical column, as `SqlDriver.syncDeclaredIndexes` asks + * before it creates one. A missing column is the same durability failure + * there and here: the index the metadata declares is not created. + */ + private async remoteIndexColumnsPresent(table: string, indexName: string, columns: readonly string[]): Promise { + const physical = new Set((await this.introspectColumns(table)).map((c) => c.name)); + const missing = columns.filter((c) => !physical.has(c)); + if (missing.length === 0) return true; + this.logDurabilityFailure( + `[schema-drift] index '${indexName}' on '${table}' was NOT created: no column for ` + + `${missing.map((c) => `'${c}'`).join(', ')} on the remote database. The constraint or access path it ` + + 'declares does not exist; fix the metadata so every key column is a stored field, then re-run "os migrate plan".', + ); + return false; } /** From 15fdbc228addd8c72170256711b6bd527521a9d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 07:04:11 +0000 Subject: [PATCH 02/10] feat(cli): the organization-ownership ceremony boots with no schema sync on a transport that cannot defer DDL bootSchemaStack gains deferredDdlUnsupported: 'skip-schema-sync'. When the primary datasource resolves to the Turso REMOTE transport (asked of TursoDriver.detectMode), the boot passes skipSchemaSync to the standalone stack instead of arming deferral on that driver: no DDL is attempted, every object reaches its driver through the DDL-free registerObjectMetadata, and every other datasource is still armed. measureComposedCoverage binds the composed host's objects DDL-free on such a boot. The ceremony's plan and apply declare the posture, and the apply reads the ledger's presence from the catalog when the boot measured no table absent. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01Lr3rg6qTp9r38exA5DHEEH --- .../migrate/organization-ownership.ts | 26 ++++- packages/cli/src/utils/schema-migrate.ts | 105 +++++++++++++++++- .../cli/src/utils/schema-migration-plugins.ts | 20 +++- packages/runtime/src/standalone-stack.ts | 18 +++ 4 files changed, 156 insertions(+), 13 deletions(-) diff --git a/packages/cli/src/commands/migrate/organization-ownership.ts b/packages/cli/src/commands/migrate/organization-ownership.ts index 1750f1b56aa..5cc02a632d6 100644 --- a/packages/cli/src/commands/migrate/organization-ownership.ts +++ b/packages/cli/src/commands/migrate/organization-ownership.ts @@ -75,6 +75,11 @@ async function confirm(question: string): Promise { * The boot is the family's read-only boot (`deferSchemaDdl` + `readOnlyProbe`: * no DDL, no seed, no database file brought into existence), and the plan * itself issues SELECT statements only (`../../utils/organization-ownership-plan.ts`). + * On a datasource whose transport cannot defer DDL (the Turso REMOTE + * transport) the boot skips schema sync instead (`deferredDdlUnsupported: + * 'skip-schema-sync'`, #23816): it attempts no DDL at all, which keeps the same + * promise. The environment's own boot has run the additive sync the ceremony + * reads; the ceremony needs no table this release would create. * It writes no row or table; the one thing it writes is the plan FILE, on the * operator's disk, which the ADR asks for: "the plan is written to a file the * operator keeps". It never overwrites one — a kept plan is evidence. @@ -149,8 +154,11 @@ async function confirm(question: string): Promise { * served platform) so the security catalog it verifies names against is the * one the deployment serves, and the schema-drift ceremony proposes the * column drops of exactly the objects this release declares without the - * column. It still holds the boot's own DDL back (`deferSchemaDdl`): the only - * schema change the ceremony makes is the drops it asked for. + * column. It still holds the boot's own DDL back (`deferSchemaDdl`, or on the + * Turso REMOTE transport no schema sync at all, as the plan boots): the only + * schema change the ceremony makes is the drops it asked for. On that transport + * the drops go through the remote face's own drift detector and migration + * door (`@objectstack/driver-turso`). */ export default class MigrateOrganizationOwnership extends Command { static override description = @@ -335,6 +343,9 @@ export default class MigrateOrganizationOwnership extends Command { jsonOutput: json, ...(flags['database-url'] ? { databaseUrl: flags['database-url'] } : {}), deferSchemaDdl: true, + // [#23816] The plan needs no held-back work listed, so a transport that + // cannot defer DDL boots with no schema sync instead (still read-only). + deferredDdlUnsupported: 'skip-schema-sync', readOnlyProbe: true, }); } catch (error: unknown) { @@ -508,6 +519,8 @@ export default class MigrateOrganizationOwnership extends Command { jsonOutput: json, ...(flags['database-url'] ? { databaseUrl: flags['database-url'] } : {}), deferSchemaDdl: true, + // [#23816] As the plan boots: no schema sync on a transport that cannot defer DDL. + deferredDdlUnsupported: 'skip-schema-sync', composeHostStack: true, composeServedPlatform: true, composeAuthGatedSecurity: true, @@ -530,7 +543,13 @@ export default class MigrateOrganizationOwnership extends Command { // The dialect, once, before the first statement: an unresolved one is // refused here, naming the driver, and never read as SQLite. const dialect = ceremonyDialectOf(stack.driver); - if (stack.tableAbsent('sys_migration')) { + const { normalizeRows } = await import('@objectstack/metadata-protocol'); + // [#23816] A boot that skipped schema sync measured no table absent, so + // the ledger's presence is read from the catalog instead. + const ledgerAbsent = stack.schemaSync === 'skipped' + ? normalizeRows(await exec(columnListSql(dialect), ['sys_migration'])).length === 0 + : stack.tableAbsent('sys_migration'); + if (ledgerAbsent) { await refuse( { reason: 'ledger-absent', table: 'sys_migration', detail: 'the deployment ledger the ceremony checkpoints into does not exist' }, 'Refused: this database has no sys_migration table, where the ceremony records its checkpoint. Boot this ' + @@ -538,7 +557,6 @@ export default class MigrateOrganizationOwnership extends Command { ); return; } - const { normalizeRows } = await import('@objectstack/metadata-protocol'); const { createSecurityCatalogReader } = await import('@objectstack/core'); const { readDataMigrationFlag, recordDataMigrationRun } = await import('@objectstack/platform-objects/system'); const kernel = stack.kernel as { getService(name: string): unknown }; diff --git a/packages/cli/src/utils/schema-migrate.ts b/packages/cli/src/utils/schema-migrate.ts index b1fcdf46db7..651b53dca0c 100644 --- a/packages/cli/src/utils/schema-migrate.ts +++ b/packages/cli/src/utils/schema-migrate.ts @@ -135,6 +135,13 @@ export interface SchemaStack { * migrate plan` / `apply` print it beside the database in both faces. */ dbSource: DatabaseSource; + /** + * [#23816] What this boot did with schema sync: `'deferred'` (held back, see + * {@link pendingSchemaWork}), `'skipped'` (none attempted, the boot's + * `deferredDdlUnsupported: 'skip-schema-sync'` answer for a transport that + * cannot defer), or `'performed'` (the ordinary additive sync ran). + */ + schemaSync: 'deferred' | 'skipped' | 'performed'; managedTableCount: number; /** The booted kernel — `getService('objectql')` etc. for one-shot commands * beyond schema migration. */ @@ -380,6 +387,17 @@ class DeferSchemaDdlPlugin { /** Every driver this boot deferred, in the order it was armed. */ readonly drivers: SqlDriverLike[] = []; + /** + * [#23816] `exemptPrimary`: leave the engine's DEFAULT driver un-armed. Set + * only by a `skipSchemaSync` boot (`deferredDdlUnsupported: + * 'skip-schema-sync'`), where the engine sends that driver no DDL at all and + * its transport would refuse the arm. Every other driver is armed as before. + */ + constructor(private readonly options: { exemptPrimary?: boolean } = {}) {} + + /** The driver {@link options}.exemptPrimary leaves un-armed, once `init()` has found it. */ + private exempt: unknown = null; + /** Engines whose `registerDriver` is shadowed, and what to restore. */ private readonly shadows: Array<{ engine: Record; @@ -388,6 +406,7 @@ class DeferSchemaDdlPlugin { }> = []; init = async (ctx: any) => { + if (this.options.exemptPrimary) this.exempt = defaultDriverOf(ctx); for (const name of SQL_DRIVER_SERVICES) { try { this.arm(ctx.getService(name)); } catch { /* not registered */ } } @@ -410,6 +429,7 @@ class DeferSchemaDdlPlugin { /** Arm one driver. Idempotent; a non-SQL driver has nothing to defer. */ arm(driver: unknown): void { if (!driver || typeof driver !== 'object') return; + if (driver === this.exempt) return; const d = driver as SqlDriverLike; if (this.drivers.includes(d)) return; if (typeof d.setDeferredDdl === 'function') { @@ -478,6 +498,43 @@ class DeferSchemaDdlPlugin { } } +/** + * [#23816] The engine's DEFAULT driver — the primary datasource's — read + * through the accessors the engine makes public, or `null` when the engine + * names none yet. + */ +function defaultDriverOf(ctx: { getService?: (name: string) => unknown }): unknown { + for (const name of ENGINE_SERVICES) { + let engine: { getDefaultDriverName?: () => unknown; getDriverByName?: (name: string) => unknown } | undefined; + try { engine = ctx.getService?.(name) as typeof engine; } catch { /* not registered */ } + const driverName = engine?.getDefaultDriverName?.(); + if (typeof driverName === 'string') return engine?.getDriverByName?.(driverName) ?? null; + } + return null; +} + +/** + * [#23816] Does the transport the boot will open for `resolved` refuse + * deferred schema DDL? Today exactly the Turso REMOTE transport. The + * standalone stack builds a `turso` datasource as `{ url, authToken }`, so + * `TursoDriver.detectMode` reads the resolved URL exactly as the driver the + * boot constructs will: the driver package answers, not a copy of its rule. + * + * Loaded lazily, as every driver is from this module (#5726). A package that + * cannot be loaded answers `false`, and the boot then fails where it always + * has, in `@objectstack/runtime`'s loader, naming the package to install. + */ +async function transportRefusesDeferredDdl(resolved: { driver: string; url: string }): Promise { + if (resolved.driver !== 'turso') return false; + let detectMode: ((config: { url: string }) => string) | undefined; + try { + detectMode = (await import('@objectstack/driver-turso')).TursoDriver.detectMode; + } catch { + return false; + } + return detectMode({ url: resolved.url }) === 'remote'; +} + /** * Name the database the migrate/resync commands are about to write to. * @@ -737,6 +794,32 @@ export async function bootSchemaStack( * of the data commands) leaves this off. */ deferSchemaDdl?: boolean; + /** + * [#23816] With {@link deferSchemaDdl}: what the boot does when the + * PRIMARY datasource's transport cannot defer schema DDL — the Turso REMOTE + * transport (`TursoDriver.detectMode` answers `remote` for the URL this boot + * resolved), whose driver refuses the deferral. + * + * - `'refuse'` (the default): arm the deferral anyway, so the boot fails + * with the driver's own refusal. Right for every command whose dry run + * lists the held-back work (`os migrate plan` / `apply`): a boot that + * holds nothing back cannot list it. + * - `'skip-schema-sync'`: boot with schema sync SKIPPED instead + * (`skipSchemaSync`, the posture `OS_SKIP_SCHEMA_SYNC=1` names). No DDL is + * held back because none is attempted, and each object is told to its + * driver DDL-free (`registerObjectMetadata`), so drift detection still + * diffs the declared set. The boot reports it ({@link SchemaStack.schemaSync}), + * {@link SchemaStack.pendingSchemaWork} is `[]` and + * {@link SchemaStack.tableAbsent} measures nothing. Only a command that + * needs no table this release would CREATE may ask for it; today that is + * `os migrate organization-ownership`, which runs against an environment + * whose own boot has already run the additive sync. + * + * Decided from the resolved URL before anything connects, never by catching + * the driver's refusal, and only for the primary: any other datasource the + * boot connects is still armed, and refuses as before if it cannot defer. + */ + deferredDdlUnsupported?: 'refuse' | 'skip-schema-sync'; /** * Boot WITHOUT BRINGING A DATABASE INTO EXISTENCE (#6743). * @@ -836,17 +919,23 @@ export async function bootSchemaStack( const envLoad = loadProjectEnvFiles(projectRoot, { dev: opts.serveFlags?.dev === true }); const { createStandaloneStack, Runtime, resolveStandaloneDatabase } = await import('@objectstack/runtime'); - const defer = opts.deferSchemaDdl === true; // The inputs the database is resolved from — by `createStandaloneStack` // below, the occupancy probe before it, and the telemetry sibling's key. const databaseInput = { projectRoot, ...(opts.databaseUrl ? { databaseUrl: opts.databaseUrl } : {}) }; + const resolvedDatabase = resolveStandaloneDatabase(databaseInput); + + // [#23816] Deferred, skipped, or performed: see `deferredDdlUnsupported`. + const skipSchemaSync = opts.deferSchemaDdl === true + && opts.deferredDdlUnsupported === 'skip-schema-sync' + && (await transportRefusesDeferredDdl(resolvedDatabase)); + const defer = opts.deferSchemaDdl === true && !skipSchemaSync; // [#22581] Who named the database: the resolution `createStandaloneStack` // makes below (and the occupancy probe made before it), from the same inputs. const dbSource = resolveDatabaseSource( envLoad, - resolveStandaloneDatabase(databaseInput), + resolvedDatabase, opts.databaseUrl || undefined, ); @@ -898,6 +987,8 @@ export async function bootSchemaStack( runPlatformMigrations: false, // [#22371] Off only when the caller asks; see the option. ...(opts.hydrateMetadata === false ? { hydrateMetadataFromDb: false } : {}), + // [#23816] See `deferredDdlUnsupported`. + ...(skipSchemaSync ? { skipSchemaSync: true } : {}), }); // No HTTP, no cluster — this is a one-shot schema operation. @@ -931,7 +1022,9 @@ export async function bootSchemaStack( warn: (message) => console.warn(message), oneShot: { readOnlyProbe: opts.readOnlyProbe === true }, })) ?? null; - const deferral = defer ? new DeferSchemaDdlPlugin() : null; + // [#23816] A skip-sync boot still arms every driver but the primary: any + // other datasource's DDL stays held back, as on a deferred boot. + const deferral = defer || skipSchemaSync ? new DeferSchemaDdlPlugin({ exemptPrimary: skipSchemaSync }) : null; if (deferral) { await kernel.use(deferral as any); } @@ -1005,7 +1098,7 @@ export async function bootSchemaStack( // so instead of reading as coverage. Runs only when this boot actually // composed a host; every other caller is untouched. if (opts.composeHostStack === true && composition.notes.length > 0) { - const measured = await measureComposedCoverage(kernel, plannedDrivers, defer); + const measured = await measureComposedCoverage(kernel, plannedDrivers, defer, { ddlFree: skipSchemaSync }); composition.coverage = measured.coverage; composition.notes.push(...measured.notes); } @@ -1022,7 +1115,8 @@ export async function bootSchemaStack( } // [#23022] …and the one table outside the object set, `sys_packages`, which // no driver deferral can see. Read-only here; `flushSchemaDdl` creates it. - let packageTable = deferral ? await planPackageTable(kernel, composition.notes) : null; + // Not on a skip-sync boot: the table lives on the primary, which held nothing back. + let packageTable = defer ? await planPackageTable(kernel, composition.notes) : null; if (packageTable) pendingSchemaWork.push(packageTable.work); sortPendingSchemaWork(pendingSchemaWork); // [#21529] The tables the held-back sync would create: measured absent. @@ -1037,6 +1131,7 @@ export async function bootSchemaStack( driver, dbLabel: describeDb(driver), dbSource, + schemaSync: defer ? 'deferred' : skipSchemaSync ? 'skipped' : 'performed', telemetryDatabase: telemetryDriver ? telemetryDatabase : null, drivers: plannedDrivers, detectManagedDrift: async (): Promise => { diff --git a/packages/cli/src/utils/schema-migration-plugins.ts b/packages/cli/src/utils/schema-migration-plugins.ts index 7dac64a970f..f4cb0c5a2eb 100644 --- a/packages/cli/src/utils/schema-migration-plugins.ts +++ b/packages/cli/src/utils/schema-migration-plugins.ts @@ -2350,12 +2350,18 @@ export function refuseWhenHostConfigUnloadable( * (#22579): an object routed to the sibling is examined there, not reported * as bound to a different datasource. * @param deferred whether this boot armed deferred DDL. `false` reports the - * coverage as UNMEASURED instead of syncing. + * coverage as UNMEASURED instead of syncing, unless `ddlFree` is set. + * @param options.ddlFree [#23816] the boot SKIPPED schema sync instead + * (`skipSchemaSync`, for a primary whose transport cannot defer DDL): each + * object is bound through the driver's DDL-free `registerObjectMetadata`, + * the binding the engine's own skip-sync `start()` uses, so nothing can run + * DDL either way. A driver without that member counts as `unsupported`. */ export async function measureComposedCoverage( kernel: unknown, plannedDrivers: readonly unknown[], deferred: boolean, + options: { ddlFree?: boolean } = {}, ): Promise<{ coverage: SchemaMigrationCoverage; notes: string[] }> { const notes: string[] = []; const empty: SchemaMigrationCoverage = { @@ -2365,7 +2371,8 @@ export async function measureComposedCoverage( reasons: { federated: 0, unbound: 0, unsupported: 0, otherDriver: 0, failed: 0 }, }; - if (!deferred) { + const ddlFree = !deferred && options.ddlFree === true; + if (!deferred && !ddlFree) { notes.push( 'This boot did not defer schema DDL, so the declared object set was NOT bound to the driver here — ' + 'binding it would have run DDL. Coverage below is UNMEASURED, not full.', @@ -2418,9 +2425,14 @@ export async function measureComposedCoverage( const driver = engine.getDriverForObject(obj.name); if (!driver) { unbound++; continue; } if (!plannedDrivers.includes(driver)) { otherDriver++; continue; } - if (typeof (driver as { syncSchema?: unknown }).syncSchema !== 'function') { unsupported++; continue; } + const register = (driver as { registerObjectMetadata?: unknown }).registerObjectMetadata; + if (ddlFree ? typeof register !== 'function' : typeof (driver as { syncSchema?: unknown }).syncSchema !== 'function') { + unsupported++; + continue; + } try { - await engine.syncObjectSchema(obj.name); + if (ddlFree) (register as (objects: unknown[]) => void).call(driver, [obj]); + else await engine.syncObjectSchema(obj.name); examined++; } catch (e: unknown) { failed++; diff --git a/packages/runtime/src/standalone-stack.ts b/packages/runtime/src/standalone-stack.ts index 787073f6c5e..38f9d0e1890 100644 --- a/packages/runtime/src/standalone-stack.ts +++ b/packages/runtime/src/standalone-stack.ts @@ -320,6 +320,21 @@ export const StandaloneStackConfigSchema = z.object({ * turning hydration ON, and this function keeps it on unless told. */ hydrateMetadataFromDb: z.boolean().optional(), + /** + * [#23816] Boot with schema sync SKIPPED: the `ObjectQLPlugin` option the + * `OS_SKIP_SCHEMA_SYNC=1` posture names, passed explicitly. The engine then + * runs no CREATE TABLE / ALTER TABLE and tells each driver its objects + * DDL-free (`registerObjectMetadata`), which is all the coercion registries + * and drift detection need. + * + * Unset, the plugin keeps reading `OS_SKIP_SCHEMA_SYNC` exactly as before, + * so every serving boot is unchanged. `bootSchemaStack` sets it for one + * case only: a command that asked for deferred DDL against a datasource + * whose transport cannot defer it (the Turso REMOTE transport), and that + * declared that a boot with no schema sync serves it as well + * (`os migrate organization-ownership`). + */ + skipSchemaSync: z.boolean().optional(), }); export type StandaloneStackConfig = z.input; @@ -964,6 +979,9 @@ export async function createStandaloneStack(config?: StandaloneStackConfig): Pro environmentId, runPlatformMigrations: cfg.runPlatformMigrations ?? true, hydrateMetadataFromDb: cfg.hydrateMetadataFromDb !== false, + // [#23816] Only a `true` is passed through, so a serving boot keeps + // the plugin's own `OS_SKIP_SCHEMA_SYNC` reading. + ...(cfg.skipSchemaSync === true ? { skipSchemaSync: true } : {}), // [#21391] Only a `false` is passed through, so a serving boot // hands the plugin exactly the options it always did. ...(cfg.armLifecycleSweep === false ? { lifecycle: { enabled: false } } : {}), From d8bb51ea77c93f125c0ecf8058da0e60e3581a24 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 07:13:02 +0000 Subject: [PATCH 03/10] test(driver-turso): the remote face's drift report and migration entries, against the local face over one file The drift-detection refusal suite becomes the detection suite: a clean remote-synced database drifts nowhere on either face, each drifted shape reads alike on both, and registerObjectMetadata fills the registry the no-argument call walks. A new suite applies the remote report's entries over a real libSQL client and reads each outcome back through a second connection: drop_column with its indexes, the rolled-back refusal, drop/create/recreate index, the D4 re-probe, the skipped rebuild-only entry, and convergence. The remote face's config now names its url, so the CLI's database label tells remote databases apart. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01Lr3rg6qTp9r38exA5DHEEH --- .../drivers/driver-turso/src/turso-driver.ts | 11 +- ...rso-remote-drift-detection-refusal.test.ts | 181 ------------ .../src/turso-remote-drift-detection.test.ts | 222 +++++++++++++++ .../turso-remote-inherited-members.test.ts | 12 +- .../turso-remote-migration-entries.test.ts | 266 ++++++++++++++++++ 5 files changed, 499 insertions(+), 193 deletions(-) delete mode 100644 packages/drivers/driver-turso/src/turso-remote-drift-detection-refusal.test.ts create mode 100644 packages/drivers/driver-turso/src/turso-remote-drift-detection.test.ts create mode 100644 packages/drivers/driver-turso/src/turso-remote-migration-entries.test.ts diff --git a/packages/drivers/driver-turso/src/turso-driver.ts b/packages/drivers/driver-turso/src/turso-driver.ts index 826c0160c90..1316b9f8ee2 100644 --- a/packages/drivers/driver-turso/src/turso-driver.ts +++ b/packages/drivers/driver-turso/src/turso-driver.ts @@ -2002,11 +2002,20 @@ export class TursoDriver extends SqlDriver { // as `introspectSchema()` did with "no tables". Now knex refuses to run it // (`Unable to acquire a connection`), because there is no connection to // run it on. That failure is loud; it never adds a silent answer. + // + // [#23816] `url` names the DECLARED target, which `SqlDriver` keeps in + // `this.config` so `describeDriverConnection` can render it (the CLI's + // "which database" label, credential-free: it drops userinfo and the query, + // where a libSQL token can ride). Knex reads no `url` key, so it opens + // nothing. Without it every remote database was labelled `better-sqlite3`, + // and the organization-ownership ceremony's refusal of a plan reviewed + // against another database could not tell two remote databases apart. if (mode === 'remote') { return { client: 'better-sqlite3', useNullAsDefault: true, - }; + url: config.url, + } as SqlDriverConfig; } if (config.url === ':memory:') { diff --git a/packages/drivers/driver-turso/src/turso-remote-drift-detection-refusal.test.ts b/packages/drivers/driver-turso/src/turso-remote-drift-detection-refusal.test.ts deleted file mode 100644 index bb891b962ce..00000000000 --- a/packages/drivers/driver-turso/src/turso-remote-drift-detection-refusal.test.ts +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. - -/** - * #19845 — schema drift detection on the Turso REMOTE face refuses instead of - * answering "no drift". - * - * `SqlDriver.detectManagedDrift` reads the physical schema through Knex, and a - * remote `TursoDriver` was then built with a placeholder `:memory:` Knex - * connection that held none of the datasource's tables (since #20054 it is - * built with no Knex connection at all). Before the refusal, the remote answer - * was `[]` for every database. That is the answer the artifact-pinned - * boot gate of `os serve` reads as "never drifted", so a gate whose job is to - * refuse a boot on destructive drift let every remote-Turso boot through. - * - * ## The reproduction, as measured before the refusal existed - * - * One declared object `t` with one field. Its table is synced, then an extra - * physical column `legacy` the declaration omits is added on disk. - * - * | face | call | answer | - * |:--|:--|:--| - * | local (`:memory:` url, Knex) | `detectManagedDrift()` | `t.legacy`: `unmapped_column`, op `drop_column`, `destructive` | - * | remote (batch door, the engine's boot sync) | `detectManagedDrift()` | `[]` | - * | remote | `detectManagedDrift([{ name: 't', fields }])` | `[]` | - * - * The remote physical table held `id, created_at, updated_at, name, legacy` - * at the time, so the drift was on disk and the detector did not see it. - * - * ## What is pinned - * - * 1. The local and embedded-replica faces still report the extra column. They - * are the controls: the same physical state, the same declaration, a real - * finding. - * 2. The remote face refuses both call shapes with `NOT_IMPLEMENTED` / `501` - * and the operator-facing first sentence, and sends nothing to the - * database while refusing. - */ - -import { describe, it, expect, afterAll } from 'vitest'; -import { TursoDriver } from './turso-driver.js'; -import { makeLibsqlSqliteStub, type LibsqlSqliteStub } from './libsql-sqlite-stub.testkit.js'; -import { replicaFiles } from './replica-file.testkit.js'; - -// A replica is a local FILE: the constructor refuses one on `:memory:`. -const replicaFileUrls = replicaFiles(); -afterAll(() => replicaFileUrls.removeAll()); - -interface WireBearingError extends Error { - code?: string; - status?: number; -} - -/** - * The refusal's opening sentence: the operator contract, because the boot gate - * prints the driver's message inside its warning. Spelled out here rather than - * imported, since a test that imports the string it asserts pins nothing about - * the wording. The rest of the message is prose that may be improved without a - * test edit. - */ -const REFUSAL_FIRST_SENTENCE = - "Schema drift detection is not supported by the Turso REMOTE transport (this datasource's " + - "transport mode is `remote`), so this driver cannot say whether the database's physical " + - 'schema matches the declared objects.'; - -const T_FIELDS = { name: { type: 'text' } }; -const T_OBJECTS = [{ name: 't', fields: T_FIELDS }]; -const ADD_LEGACY = 'ALTER TABLE "t" ADD COLUMN "legacy" TEXT'; - -/** The finding the local detector reports for the extra column. */ -const LEGACY_FINDING = { - table: 't', - column: 'legacy', - kind: 'unmapped_column', - category: 'destructive', - op: { type: 'drop_column' }, -}; - -const sqlOf = (stmt: unknown): string => - typeof stmt === 'string' ? stmt : String((stmt as { sql?: unknown }).sql ?? ''); - -/** Wrap the stub so every statement the transport sends is recorded. */ -function record(stub: LibsqlSqliteStub) { - const statements: string[] = []; - const client = { - async execute(stmt: unknown) { - statements.push(sqlOf(stmt)); - return stub.execute(stmt); - }, - async batch(stmts: unknown[]) { - for (const s of stmts) statements.push(sqlOf(s)); - return stub.batch(stmts); - }, - close() { - stub.close(); - }, - }; - return { client, statements }; -} - -const columnsOf = (stub: LibsqlSqliteStub, table: string) => - (stub.raw.prepare(`pragma table_info("${table}")`).all() as Array<{ name: string }>).map((r) => r.name); - -/** A remote driver whose `t` was synced by the engine's boot-sync door, then drifted on disk. */ -async function driftedRemote() { - const stub = makeLibsqlSqliteStub(); - const rec = record(stub); - const driver = new TursoDriver({ url: 'libsql://drift.turso.io', client: rec.client as never }); - await driver.connect(); - expect(driver.transportMode).toBe('remote'); - await driver.syncSchemasBatch([{ object: 't', schema: { name: 't', fields: T_FIELDS } }]); - stub.raw.prepare(ADD_LEGACY).run(); - // Non-vacuous: the drift is on disk, so an empty answer would be a miss. - expect(columnsOf(stub, 't')).toContain('legacy'); - return { driver, rec }; -} - -type Call = 'no arguments' | 'explicit objects'; -const CALLS: Record ReturnType> = { - // The call the artifact-pinned boot gate makes. - 'no arguments': (driver) => driver.detectManagedDrift(), - 'explicit objects': (driver) => driver.detectManagedDrift(T_OBJECTS), -}; - -describe('controls — the Knex detector still reports the extra column', () => { - it.each(['no arguments', 'explicit objects'])('local face, %s', async (call) => { - const driver = new TursoDriver({ url: ':memory:' }); - await driver.connect(); - expect(driver.transportMode).toBe('local'); - await driver.initObjects(T_OBJECTS); - await driver.execute(ADD_LEGACY); - - const drift = await CALLS[call](driver); - - expect(drift).toEqual([expect.objectContaining({ ...LEGACY_FINDING, op: expect.objectContaining(LEGACY_FINDING.op) })]); - await driver.disconnect(); - }); - - it.each(['no arguments', 'explicit objects'])('embedded-replica face, %s', async (call) => { - // Replica mode reads its local file through Knex; `sync.onConnect: false` - // keeps the (stubbed) sync target out of the measurement. - const stub = makeLibsqlSqliteStub(); - const driver = new TursoDriver({ - url: replicaFileUrls.next(), - syncUrl: 'libsql://drift.turso.io', - client: record(stub).client as never, - sync: { onConnect: false }, - }); - await driver.connect(); - expect(driver.transportMode).toBe('replica'); - await driver.initObjects(T_OBJECTS); - await driver.execute(ADD_LEGACY); - - const drift = await CALLS[call](driver); - - expect(drift).toEqual([expect.objectContaining({ ...LEGACY_FINDING, op: expect.objectContaining(LEGACY_FINDING.op) })]); - await driver.disconnect(); - }); -}); - -describe('remote face — refused, never "no drift"', () => { - it.each(['no arguments', 'explicit objects'])( - 'refuses with NOT_IMPLEMENTED / 501, %s, and sends nothing to the database', - async (call) => { - const { driver, rec } = await driftedRemote(); - const sentBefore = rec.statements.length; - - const err = await CALLS[call](driver).then( - (drift) => { - throw new Error(`expected a refusal, got an answer: ${JSON.stringify(drift)}`); - }, - (e: unknown) => e as WireBearingError, - ); - - expect(err.code).toBe('NOT_IMPLEMENTED'); - expect(err.status).toBe(501); - expect(err.message.startsWith(REFUSAL_FIRST_SENTENCE)).toBe(true); - expect(rec.statements.slice(sentBefore)).toEqual([]); - await driver.disconnect(); - }, - ); -}); diff --git a/packages/drivers/driver-turso/src/turso-remote-drift-detection.test.ts b/packages/drivers/driver-turso/src/turso-remote-drift-detection.test.ts new file mode 100644 index 00000000000..f4ff51d00d8 --- /dev/null +++ b/packages/drivers/driver-turso/src/turso-remote-drift-detection.test.ts @@ -0,0 +1,222 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #23816 — schema drift detection on the Turso REMOTE face answers, as the + * local face answers it. + * + * Since #19845 the remote face refused `detectManagedDrift()` with + * `NOT_IMPLEMENTED` / 501, because the inherited detector read the physical + * schema through a Knex instance remote mode builds with no connection, and + * walked a registry no remote schema door filled (its answer had been `[]` for + * every database). The refusal left the v18 organization-ownership ceremony + * with no remote path: its column drops go through this detector. + * + * ## The instrument + * + * The remote face is a `TursoDriver` in remote mode over a real + * `@libsql/client` `file:` client, so its statements run on a real libSQL + * connection. The CONTROL is the local face (Knex) opened on the SAME file, + * given the same declarations DDL-free (`registerObjectMetadata`). Every + * remote answer is asserted EQUAL to the local face's answer over the same + * bytes, not to a hand-written expectation, so the two read paths (the remote + * catalog reads and the Knex introspection) are held to one verdict. + * + * ## What is pinned + * + * 1. A clean table synced by each of the three remote schema doors drifts + * nowhere, on either face. This is what the `os serve` boot gate reads: a + * false finding here would refuse a healthy boot. + * 2. Each drifted shape (an extra physical column, a declared index the table + * lacks, a bare organization key a NULL-safe unique must tighten, over + * clean and over duplicate rows) reads alike on both faces, with or without + * explicit objects. + * 3. The DDL-free `registerObjectMetadata` (a `skipSchemaSync` boot) fills the + * registry the no-argument call walks, and sends nothing to the database. + * 4. A declared table the database does not hold is skipped, as locally. + */ + +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, afterEach, describe, expect, it } from 'vitest'; +import { createClient, type Client } from '@libsql/client'; +import { TursoDriver } from './turso-driver.js'; + +const scratch = mkdtempSync(join(tmpdir(), 'turso-remote-drift-')); +afterAll(() => rmSync(scratch, { recursive: true, force: true })); +let seq = 0; +const nextFile = () => join(scratch, `db-${++seq}.db`); + +const open: Array<{ driver: TursoDriver; client: Client | null }> = []; +afterEach(async () => { + while (open.length) { + const { driver, client } = open.pop()!; + await driver.disconnect().catch(() => {}); + client?.close(); + } +}); + +type Objects = Parameters[0] & object; + +const sqlOf = (stmt: unknown): string => (typeof stmt === 'string' ? stmt : String((stmt as { sql?: unknown }).sql)); + +/** A remote face over `file`, every statement it sends recorded. */ +async function remoteOver(file: string) { + const client = createClient({ url: `file:${file}` }); + const statements: string[] = []; + const recording = { + execute: (stmt: unknown) => { + statements.push(sqlOf(stmt)); + return client.execute(stmt as never); + }, + batch: (stmts: unknown[], mode?: unknown) => { + for (const s of stmts) statements.push(sqlOf(s)); + return client.batch(stmts as never, mode as never); + }, + close: () => client.close(), + } as unknown as Client; + const driver = new TursoDriver({ url: 'libsql://drift.example.turso.io', client: recording }); + open.push({ driver, client }); + await driver.connect(); + expect(driver.transportMode).toBe('remote'); + return { driver, statements }; +} + +/** The local (Knex) face over the same file, told the declarations DDL-free. */ +async function localOver(file: string, objects: Objects): Promise { + const driver = new TursoDriver({ url: `file:${file}` }); + open.push({ driver, client: null }); + await driver.connect(); + expect(driver.transportMode).toBe('local'); + driver.registerObjectMetadata(objects); + return driver; +} + +/** One statement on the file, through a connection of its own. */ +async function onDisk(file: string, sql: string): Promise { + const c = createClient({ url: `file:${file}` }); + try { + await c.execute(sql); + } finally { + c.close(); + } +} + +const T = { name: 't', fields: { name: { type: 'text' }, n: { type: 'number' }, flag: { type: 'boolean' } } }; +const TENANT = { + name: 'tenant_t', + fields: { code: { type: 'text', unique: true }, label: { type: 'text' }, organization_id: { type: 'text' } }, +}; +const INDEXED = { name: 'indexed_t', fields: { name: { type: 'text' } }, indexes: [{ fields: ['name'] }] }; +const ALL: Objects = [T, TENANT, INDEXED]; + +type Door = 'syncSchemasBatch' | 'syncSchema' | 'initObjects'; +const SYNC: Record Promise> = { + syncSchemasBatch: (d, objects) => d.syncSchemasBatch(objects.map((o) => ({ object: o.name, schema: o }))), + syncSchema: async (d, objects) => { + for (const o of objects) await d.syncSchema(o.name, o); + }, + initObjects: (d, objects) => d.initObjects(objects as never), +}; + +describe('a clean remote-synced database drifts nowhere, on either face', () => { + it.each(['syncSchemasBatch', 'syncSchema', 'initObjects'])('synced through %s', async (door) => { + const file = nextFile(); + const { driver: remote } = await remoteOver(file); + await SYNC[door](remote, ALL); + const local = await localOver(file, ALL); + + expect(await local.detectManagedDrift(ALL)).toEqual([]); + expect(await remote.detectManagedDrift(ALL)).toEqual([]); + // The call the boot gate makes: the registry every remote door now fills. + expect(await remote.detectManagedDrift()).toEqual([]); + }); +}); + +type Drift = Awaited>; + +describe('each drifted shape reads alike on the remote and the local face', () => { + /** The physical drift each case puts on disk after a clean remote sync, and the finding it must produce. */ + const CASES: Array<[string, string[], (drift: Drift) => void]> = [ + [ + 'an extra physical column the declaration omits', + ['ALTER TABLE "t" ADD COLUMN "legacy" TEXT', 'CREATE INDEX "idx_t_legacy" ON "t" ("legacy")'], + (drift) => expect(drift).toContainEqual(expect.objectContaining({ + table: 't', column: 'legacy', category: 'destructive', op: expect.objectContaining({ type: 'drop_column' }), + })), + ], + [ + 'a declared index the table lacks', + ['DROP INDEX "idx_indexed_t_name"'], + (drift) => expect(drift).toContainEqual(expect.objectContaining({ + table: 'indexed_t', op: expect.objectContaining({ type: 'create_index', indexName: 'idx_indexed_t_name' }), + })), + ], + [ + 'a bare organization key a NULL-safe unique tightens, over clean rows', + [ + 'DROP INDEX "uniq_tenant_t_organization_id_code"', + 'CREATE UNIQUE INDEX "uniq_tenant_t_organization_id_code" ON "tenant_t" ("organization_id", "code")', + ], + (drift) => expect(drift).toContainEqual(expect.objectContaining({ + table: 'tenant_t', category: 'safe', op: expect.objectContaining({ type: 'recreate_index', tightenNullSafeOnly: true }), + })), + ], + [ + 'the same tightening over duplicate rows the bare key admitted', + [ + 'DROP INDEX "uniq_tenant_t_organization_id_code"', + 'CREATE UNIQUE INDEX "uniq_tenant_t_organization_id_code" ON "tenant_t" ("organization_id", "code")', + `INSERT INTO "tenant_t" ("id", "code", "organization_id") VALUES ('a', 'X', NULL), ('b', 'X', NULL)`, + ], + (drift) => expect(drift).toContainEqual(expect.objectContaining({ + table: 'tenant_t', category: 'destructive', op: expect.objectContaining({ type: 'recreate_index' }), + })), + ], + ]; + + it.each(CASES)('%s', async (_label, statements, finding) => { + const file = nextFile(); + const { driver: remote } = await remoteOver(file); + await SYNC.syncSchemasBatch(remote, ALL); + for (const sql of statements) await onDisk(file, sql); + const local = await localOver(file, ALL); + + const expected = await local.detectManagedDrift(ALL); + // Non-vacuous: the local face sees the drift the case put on disk. + finding(expected); + expect(await remote.detectManagedDrift(ALL)).toEqual(expected); + expect(await remote.detectManagedDrift()).toEqual(expected); + }); +}); + +describe('the registry the no-argument call walks', () => { + it('a skipSchemaSync boot fills it through registerObjectMetadata, sending nothing', async () => { + const file = nextFile(); + const { driver: synced } = await remoteOver(file); + await SYNC.syncSchemasBatch(synced, [T]); + await onDisk(file, 'ALTER TABLE "t" ADD COLUMN "legacy" TEXT'); + + const { driver: remote, statements } = await remoteOver(file); + expect(await remote.detectManagedDrift()).toEqual([]); // nothing has reached this driver yet + const sent = statements.length; + remote.registerObjectMetadata([T]); + expect(statements.length).toBe(sent); + + const local = await localOver(file, [T]); + const expected = await local.detectManagedDrift([T]); + expect(expected).toContainEqual(expect.objectContaining({ table: 't', column: 'legacy' })); + expect(await remote.detectManagedDrift()).toEqual(expected); + }); + + it('a declared table the database does not hold is skipped, as on the local face', async () => { + const file = nextFile(); + const { driver: remote } = await remoteOver(file); + await SYNC.syncSchemasBatch(remote, [T]); + const absent = { name: 'never_created', fields: { x: { type: 'text' } } }; + const local = await localOver(file, [T, absent]); + + expect(await local.detectManagedDrift([T, absent])).toEqual([]); + expect(await remote.detectManagedDrift([T, absent])).toEqual([]); + }); +}); diff --git a/packages/drivers/driver-turso/src/turso-remote-inherited-members.test.ts b/packages/drivers/driver-turso/src/turso-remote-inherited-members.test.ts index c84f5bbdec9..aab6f9cac2e 100644 --- a/packages/drivers/driver-turso/src/turso-remote-inherited-members.test.ts +++ b/packages/drivers/driver-turso/src/turso-remote-inherited-members.test.ts @@ -32,7 +32,7 @@ * | `introspectSchema` | `Unable to acquire a connection` | 501 | * | `getKnex` | a Knex whose every statement fails | 501 | * | `rotateShards` | `Unable to acquire a connection` | 501 | - * | `applyMigrationEntries` | resolved, every entry `skipped` | 501 | + * | `applyMigrationEntries` | resolved, every entry `skipped` | 501; since #23816 answered (`turso-remote-migration-entries.test.ts`) | * | `reclaimSpace` | `Unable to acquire a connection` | resolves; pages returned | * | `supportsRotation` | `true` | `false` | * | `setFileColumnsMovedResolver` | `true` (never asked) | `false` | @@ -212,9 +212,6 @@ describe('members refused on the remote face, beside the local control that answ const WINDOW = { windowFunctions: [{ function: 'row_number', alias: 'rn', over: { orderBy: [{ field: 'id', order: 'asc' }] } }], } as unknown as Parameters[1]; - const DROP_N = [ - { table: 'probe_t', column: 'n', kind: 'unmapped_column', category: 'destructive', op: { type: 'drop_column' }, message: 'probe' }, - ] as unknown as Parameters[0]; const ROTATED = { name: 'probe_rot', fields: { name: { type: 'text' } }, @@ -228,19 +225,12 @@ describe('members refused on the remote face, beside the local control that answ ['introspectSchema', 'Schema introspection (`introspectSchema()`)', (d) => d.introspectSchema()], ['getKnex', 'Handing out the Knex instance (`getKnex()`)', (d) => d.getKnex()], ['rotateShards', 'Data-lifecycle shard rotation (`rotateShards()`)', (d) => d.rotateShards(ROTATED)], - ['applyMigrationEntries', 'Applying schema migration entries (`applyMigrationEntries()`)', (d) => d.applyMigrationEntries(DROP_N, { allowDestructive: true })], ])('%s', async (_member, operation, call) => { await expect(Promise.resolve(call(await seeded('local')))).resolves.toBeDefined(); const remote = await seeded('remote'); expectRefused(await refusalOf(() => call(remote)), operation); }); - it('applyMigrationEntries left the remote table as it was: the column is still read back', async () => { - const remote = await seeded('remote'); - await refusalOf(() => remote.applyMigrationEntries(DROP_N, { allowDestructive: true })); - expect((await remote.find('probe_t', { where: { id: 'a' } }))[0]).toEqual(expect.objectContaining({ n: 1 })); - }); - it('the deprecated commitTransaction / rollbackTransaction answer with the commit / rollback refusal', async () => { const remote = await seeded('remote'); const handle = { commit: async () => {}, rollback: async () => {} } as never; diff --git a/packages/drivers/driver-turso/src/turso-remote-migration-entries.test.ts b/packages/drivers/driver-turso/src/turso-remote-migration-entries.test.ts new file mode 100644 index 00000000000..da614978510 --- /dev/null +++ b/packages/drivers/driver-turso/src/turso-remote-migration-entries.test.ts @@ -0,0 +1,266 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #23816 — `applyMigrationEntries()` on the Turso REMOTE face applies a drift + * report's index entries and `drop_column` through the libSQL client. + * + * Since #20055 the member refused with `NOT_IMPLEMENTED` / 501 there (before + * that it resolved with every entry `skipped`, destructive ones its caller had + * allowed included). The v18 organization-ownership ceremony drops its columns + * through this member, so a hosted tenant on the remote transport could not + * finish the ceremony at all. + * + * ## The instrument + * + * A remote-mode `TursoDriver` over a real `@libsql/client` `file:` client: a + * real libSQL connection, so a `write` batch is the transaction it is on a + * Turso server. Each entry comes from the remote face's own drift report (the + * shape the ceremony hands it), and every outcome is read back through a + * SECOND connection on the file, never through the driver that applied it. + * + * ## What is pinned + * + * 1. `drop_column` drops the column and every index keyed on it, keeps the + * rows, and is not repeated when the column is already gone; without + * `allowDestructive` it is skipped and nothing moves. + * 2. A column SQLite refuses to drop (here a UNIQUE constraint) is skipped, + * and the indexes the same batch had dropped first are back: the batch is + * one transaction. + * 3. `drop_index`, `create_index` and `recreate_index` reach their target + * state; the NULL-safe tightening re-probes for duplicates and refuses over + * them, leaving the old index exactly as it was. + * 4. A column entry this transport cannot perform (it needs the SQLite table + * rebuild) is skipped, not reported applied. + * 5. Once the report is applied, the remote face reports no drift, and the + * local face over the same file agrees. + */ + +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, afterEach, describe, expect, it } from 'vitest'; +import { createClient, type Client } from '@libsql/client'; +import { TursoDriver } from './turso-driver.js'; + +const scratch = mkdtempSync(join(tmpdir(), 'turso-remote-migrate-')); +afterAll(() => rmSync(scratch, { recursive: true, force: true })); +let seq = 0; +const nextFile = () => join(scratch, `db-${++seq}.db`); + +const open: Array<{ driver: TursoDriver; client: Client | null }> = []; +afterEach(async () => { + while (open.length) { + const { driver, client } = open.pop()!; + await driver.disconnect().catch(() => {}); + client?.close(); + } +}); + +type Objects = Parameters[0] & object; +type Entry = Awaited>[number]; + +async function remoteOver(file: string): Promise { + const client = createClient({ url: `file:${file}` }); + const driver = new TursoDriver({ url: 'libsql://migrate.example.turso.io', client }); + open.push({ driver, client }); + await driver.connect(); + expect(driver.transportMode).toBe('remote'); + return driver; +} + +/** Read the file through a connection of its own. */ +async function secondConnection(file: string, sql: string): Promise>> { + const c = createClient({ url: `file:${file}` }); + try { + return (await c.execute(sql)).rows as unknown as Array>; + } finally { + c.close(); + } +} +async function onDisk(file: string, ...statements: string[]): Promise { + const c = createClient({ url: `file:${file}` }); + try { + for (const sql of statements) await c.execute(sql); + } finally { + c.close(); + } +} +const columnsOf = async (file: string, table: string) => + (await secondConnection(file, `SELECT name FROM pragma_table_info('${table}') ORDER BY cid`)).map((r) => String(r.name)); +const indexesOf = async (file: string, table: string) => + Object.fromEntries( + (await secondConnection(file, `SELECT name, sql FROM sqlite_master WHERE type = 'index' AND tbl_name = '${table}' AND sql IS NOT NULL`)) + .map((r) => [String(r.name), String(r.sql)]), + ); + +/** `t` as the release declares it: no `legacy`. */ +const T = { name: 't', fields: { name: { type: 'text' } } }; + +/** A remote face whose `t` carries a 17.x-style `legacy` column with its own index, and two rows. */ +async function withLegacyColumn(): Promise<{ file: string; remote: TursoDriver; drop: Entry }> { + const file = nextFile(); + const remote = await remoteOver(file); + await remote.syncSchemasBatch([{ object: 't', schema: T }]); + await onDisk( + file, + 'ALTER TABLE "t" ADD COLUMN "legacy" TEXT', + 'CREATE INDEX "idx_t_legacy" ON "t" ("legacy")', + 'CREATE INDEX "idx_t_name_legacy" ON "t" ("name", "legacy")', + `INSERT INTO "t" ("id", "name", "legacy") VALUES ('a', 'A', 'x'), ('b', 'B', NULL)`, + ); + const drop = (await remote.detectManagedDrift()).find((d) => d.op.type === 'drop_column' && d.column === 'legacy'); + expect(drop).toBeDefined(); + return { file, remote, drop: drop! }; +} + +describe('drop_column', () => { + it('drops the column and every index keyed on it, keeping the rows', async () => { + const { file, remote, drop } = await withLegacyColumn(); + expect(Object.keys(await indexesOf(file, 't')).sort()).toEqual(['idx_t_legacy', 'idx_t_name_legacy']); + + const { applied, skipped } = await remote.applyMigrationEntries([drop], { allowDestructive: true }); + + expect([applied, skipped]).toEqual([[drop], []]); + expect(await columnsOf(file, 't')).not.toContain('legacy'); + expect(await indexesOf(file, 't')).toEqual({}); + expect(await secondConnection(file, 'SELECT id, name FROM "t" ORDER BY id')).toEqual([ + { id: 'a', name: 'A' }, + { id: 'b', name: 'B' }, + ]); + // Re-applied over a column that is gone: nothing to do, so not `applied`. + expect(await remote.applyMigrationEntries([drop], { allowDestructive: true })).toEqual({ applied: [], skipped: [drop] }); + }); + + it('without allowDestructive it is skipped, and nothing moves', async () => { + const { file, remote, drop } = await withLegacyColumn(); + expect(await remote.applyMigrationEntries([drop])).toEqual({ applied: [], skipped: [drop] }); + expect(await columnsOf(file, 't')).toContain('legacy'); + expect(Object.keys(await indexesOf(file, 't')).sort()).toEqual(['idx_t_legacy', 'idx_t_name_legacy']); + }); + + it('a column SQLite refuses to drop is skipped, and the indexes dropped before it are rolled back', async () => { + const file = nextFile(); + const remote = await remoteOver(file); + await remote.syncSchemasBatch([{ object: 't', schema: T }]); + // Rebuilt with `legacy` under an inline UNIQUE constraint, plus a plain index on it. + await onDisk( + file, + 'DROP TABLE "t"', + 'CREATE TABLE "t" ("id" TEXT PRIMARY KEY, "created_at" TEXT, "updated_at" TEXT, "name" TEXT, "legacy" TEXT UNIQUE)', + 'CREATE INDEX "idx_t_legacy" ON "t" ("legacy")', + ); + const drop = (await remote.detectManagedDrift()).find((d) => d.op.type === 'drop_column' && d.column === 'legacy')!; + expect(drop).toBeDefined(); + + expect(await remote.applyMigrationEntries([drop], { allowDestructive: true })).toEqual({ applied: [], skipped: [drop] }); + expect(await columnsOf(file, 't')).toContain('legacy'); + expect(Object.keys(await indexesOf(file, 't'))).toEqual(['idx_t_legacy']); + }); +}); + +const INDEXED = { name: 'indexed_t', fields: { name: { type: 'text' } }, indexes: [{ fields: ['name'] }] }; +const TENANT = { + name: 'tenant_t', + fields: { code: { type: 'text', unique: true }, organization_id: { type: 'text' } }, +}; +const TENANT_UNIQUE = 'uniq_tenant_t_organization_id_code'; + +describe('the index entries', () => { + it('drop_index: an undeclared index is dropped, and a second run finds the step done', async () => { + const file = nextFile(); + const remote = await remoteOver(file); + await remote.syncSchemasBatch([{ object: 'indexed_t', schema: { name: 'indexed_t', fields: INDEXED.fields } }]); + await onDisk(file, 'CREATE INDEX "idx_indexed_t_name" ON "indexed_t" ("name")'); + const drop = (await remote.detectManagedDrift()).find((d) => d.op.type === 'drop_index')!; + expect(drop).toBeDefined(); + + expect(await remote.applyMigrationEntries([drop], { allowDestructive: true })).toEqual({ applied: [drop], skipped: [] }); + expect(await indexesOf(file, 'indexed_t')).toEqual({}); + expect(await remote.applyMigrationEntries([drop], { allowDestructive: true })).toEqual({ applied: [drop], skipped: [] }); + }); + + it('create_index: a declared index the table lacks is created as the schema sync creates it', async () => { + const file = nextFile(); + const remote = await remoteOver(file); + await remote.syncSchemasBatch([{ object: 'indexed_t', schema: INDEXED }]); + const synced = await indexesOf(file, 'indexed_t'); + await onDisk(file, 'DROP INDEX "idx_indexed_t_name"'); + const create = (await remote.detectManagedDrift()).find((d) => d.op.type === 'create_index')!; + expect(create).toBeDefined(); + + expect(await remote.applyMigrationEntries([create])).toEqual({ applied: [create], skipped: [] }); + expect(await indexesOf(file, 'indexed_t')).toEqual(synced); + }); + + /** A tenant table whose NULL-safe unique was left as the bare composite, optionally over duplicates. */ + async function bareTenantKey(duplicates: boolean) { + const file = nextFile(); + const remote = await remoteOver(file); + await remote.syncSchemasBatch([{ object: 'tenant_t', schema: TENANT }]); + const synced = await indexesOf(file, 'tenant_t'); + await onDisk( + file, + `DROP INDEX "${TENANT_UNIQUE}"`, + `CREATE UNIQUE INDEX "${TENANT_UNIQUE}" ON "tenant_t" ("organization_id", "code")`, + ...(duplicates ? [`INSERT INTO "tenant_t" ("id", "code", "organization_id") VALUES ('a', 'X', NULL), ('b', 'X', NULL)`] : []), + ); + const bare = await indexesOf(file, 'tenant_t'); + const recreate = (await remote.detectManagedDrift()).find((d) => d.op.type === 'recreate_index')!; + expect(recreate).toBeDefined(); + return { file, remote, synced, bare, recreate }; + } + + it('recreate_index: the bare key tightens to the NULL-safe one over clean rows', async () => { + const { file, remote, synced, recreate } = await bareTenantKey(false); + expect(await remote.applyMigrationEntries([recreate], { allowDestructive: true })).toEqual({ applied: [recreate], skipped: [] }); + expect(await indexesOf(file, 'tenant_t')).toEqual(synced); + }); + + it('recreate_index: over duplicates the re-probe refuses, and the old index is exactly as it was', async () => { + const { file, remote, bare, recreate } = await bareTenantKey(true); + expect(await remote.applyMigrationEntries([recreate], { allowDestructive: true })).toEqual({ applied: [], skipped: [recreate] }); + expect(await indexesOf(file, 'tenant_t')).toEqual(bare); + }); +}); + +describe('what this transport cannot perform, and convergence', () => { + it('a column entry that needs the SQLite table rebuild is skipped, never reported applied', async () => { + const { file, remote } = await withLegacyColumn(); + const entry = { + table: 't', column: 'name', kind: 'nullability', category: 'safe', severity: 'warning', message: 'probe', + op: { type: 'relax_not_null', table: 't', column: 'name' }, + } as unknown as Entry; + const before = await columnsOf(file, 't'); + expect(await remote.applyMigrationEntries([entry], { allowDestructive: true })).toEqual({ applied: [], skipped: [entry] }); + expect(await columnsOf(file, 't')).toEqual(before); + }); + + it('once the report is applied the remote face reports no drift, and the local face agrees', async () => { + const { file, remote } = await withLegacyColumn(); + const report = await remote.detectManagedDrift(); + expect(report.length).toBeGreaterThan(0); + const { skipped } = await remote.applyMigrationEntries(report, { allowDestructive: true }); + expect(skipped).toEqual([]); + + expect(await remote.detectManagedDrift()).toEqual([]); + const local = new TursoDriver({ url: `file:${file}` }); + open.push({ driver: local, client: null }); + await local.connect(); + local.registerObjectMetadata([T] as Objects); + expect(await local.detectManagedDrift([T] as Objects)).toEqual([]); + }); +}); + +describe('the database a migration names', () => { + // `SqlDriver` keeps the DECLARED target in `config` for + // `describeDriverConnection`, the CLI's "which database" label, which the + // organization-ownership ceremony writes into its plan and compares before + // an apply. The remote face used to keep no target there, so every remote + // database read `better-sqlite3`. + it('the remote face keeps its url in config, and still no Knex connection', () => { + const url = 'libsql://tenant-a.example.turso.io'; + const config = (new TursoDriver({ url, authToken: 'probe' }) as unknown as { config: Record }).config; + expect(config).toEqual({ client: 'better-sqlite3', useNullAsDefault: true, url }); + expect('connection' in config).toBe(false); + }); +}); From 1dae0bfb96d6ef2a05dde8a82b6cf767744c537e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 07:17:05 +0000 Subject: [PATCH 04/10] test(cli): the ceremony on a real sqld remote face (OS_TEST_SQLD_BIN), and the DDL-free coverage binding Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01Lr3rg6qTp9r38exA5DHEEH --- ...-ownership.remote-face.integration.test.ts | 250 ++++++++++++++++++ .../utils/schema-migration-plugins.test.ts | 30 +++ 2 files changed, 280 insertions(+) create mode 100644 packages/cli/src/commands/migrate/organization-ownership.remote-face.integration.test.ts diff --git a/packages/cli/src/commands/migrate/organization-ownership.remote-face.integration.test.ts b/packages/cli/src/commands/migrate/organization-ownership.remote-face.integration.test.ts new file mode 100644 index 00000000000..a38ebc28a9a --- /dev/null +++ b/packages/cli/src/commands/migrate/organization-ownership.remote-face.integration.test.ts @@ -0,0 +1,250 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #23816 — the v18 organization-ownership ceremony, plan and apply, completes + * on a Turso REMOTE database: a real `sqld` reached as + * `libsql://127.0.0.1:?tls=0`, the transport every hosted tenant uses. + * + * ## What used to happen + * + * Both boots armed deferred schema DDL unconditionally, and the remote face + * refuses that (`Deferred schema DDL is not supported by the Turso REMOTE + * transport`), so the plan exited 1 before reading a table; the column drops + * then went through `detectManagedDrift` + `applyMigrationEntries`, which the + * remote face refused too. Every hosted environment holding 17.x data stopped + * at its first v18 request. + * + * ## The case + * + * The 17.x database is built the way an environment gets one: an ordinary, + * syncing boot of the composition the ceremony boots, against the server (the + * additive sync on the remote face), then the 17.x leftovers a second libSQL + * connection adds — an `organization_id` column and the tenant index keyed on + * it, on `column-drop` tables, and rows. Then the real command, plan then + * `--apply --backup-taken`, and every claim is read back through a SECOND + * connection to the server's own database: + * + * - the plan exits 0 and leaves the database byte-identical (a logical dump of + * every catalog row and every table's rows, before and after), and names the + * remote database, not the placeholder Knex client; + * - the apply completes with `blocking 0`; + * - each column and the index keyed on it are gone from the server, the rows + * are not, and the `adr-0131-organization-ownership` marker is verified. + * + * ## How to run it + * + * It needs a `sqld` binary (libsql-server), so it is skipped unless + * `OS_TEST_SQLD_BIN` names one; the case starts its own server on a free port + * over a temporary directory and stops it afterwards. One way to get the binary + * is the npm package that ships it (`npm pack @sqld/linux-x64` or + * `@sqld/darwin-arm64`, then `tar -xzf` and point the variable at + * `package/sqld`); the `ghcr.io/tursodatabase/libsql-server` image and the + * Turso CLI's `turso dev` carry the same server. + * + * OS_TEST_SQLD_BIN=/path/to/sqld pnpm --filter @objectstack/cli exec \ + * vitest run --project integration src/commands/migrate/organization-ownership.remote-face.integration.test.ts + */ + +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { spawn, type ChildProcess } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { createServer } from 'node:net'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { TursoDriver } from '@objectstack/driver-turso'; +import { ORGANIZATION_OWNERSHIP_MIGRATION_ID } from '@objectstack/spec/system'; +import { bootSchemaStack } from '../../utils/schema-migrate.js'; +import { isExitSignal } from '../../utils/format.js'; +import MigrateOrganizationOwnership from './organization-ownership.js'; + +// Paid at module load, not inside a clocked case (`check:test-source-alias`). +import '@objectstack/runtime'; +import '@objectstack/objectql'; + +const SQLD = process.env.OS_TEST_SQLD_BIN; +const HERE = dirname(fileURLToPath(import.meta.url)); +const CLI_ROOT = resolve(HERE, '..', '..', '..'); +const CASE_TIMEOUT_MS = 240_000; + +/** Env that would point a boot somewhere other than the fixture, or change what it composes. */ +const OVERRIDING_ENV = [ + 'OS_DATABASE_URL', 'DATABASE_URL', 'TURSO_DATABASE_URL', 'TURSO_AUTH_TOKEN', 'OS_DATABASE_AUTH_TOKEN', + 'OS_DATABASE_DRIVER', 'OS_HOME', 'OS_ARTIFACT_PATH', 'OS_TELEMETRY_DB', 'OS_TENANCY_POSTURE', + 'OS_MULTI_ORG_ENABLED', 'OS_SKIP_SCHEMA_SYNC', +] as const; + +/** `column-drop` tables the 17.x leftovers are put on, when the composition has them. */ +const LEFTOVER_CANDIDATES = ['sys_job', 'sys_metadata_activation', 'sys_presence'] as const; + +let dir: string; +let projectDir: string; +let server: ChildProcess | undefined; +let url: string; +let leftovers: string[] = []; +const savedEnv: Record = {}; +const savedCwd = process.cwd(); + +const freePort = (): Promise => + new Promise((done, fail) => { + const probe = createServer(); + probe.once('error', fail); + probe.listen(0, '127.0.0.1', () => { + const { port } = probe.address() as { port: number }; + probe.close(() => done(port)); + }); + }); + +type Rows = Array>; +type Exec = (sql: string, args?: unknown[]) => Promise; + +/** + * A second connection to the server, separate from every driver the command + * boots: a remote-mode driver of its own, so its own libSQL client. + */ +async function secondConnection(read: (exec: Exec) => Promise): Promise { + const driver = new TursoDriver({ url }); + await driver.connect(); + try { + return await read(async (sql, args = []) => (await driver.execute(sql, args)) as Rows); + } finally { + await driver.disconnect(); + } +} + +/** Every catalog row and every table's rows, hashed: equal means nothing moved. */ +const logicalDump = () => + secondConnection(async (exec) => { + const hash = createHash('sha256'); + const master = await exec('SELECT type, name, tbl_name, sql FROM sqlite_master ORDER BY type, name'); + hash.update(JSON.stringify(master)); + for (const t of master.filter((r) => r.type === 'table' && !String(r.name).startsWith('sqlite_'))) { + hash.update(String(t.name)); + hash.update(JSON.stringify(await exec(`SELECT * FROM "${String(t.name)}" ORDER BY rowid`))); + } + return hash.digest('hex'); + }); + +const hasOrganizationColumn = (table: string) => + secondConnection(async (exec) => + (await exec('SELECT name FROM pragma_table_info(?)', [table])).some((r) => r.name === 'organization_id')); + +/** The real command, its stdout parsed, and the exit it asked for. */ +async function run(args: string[]): Promise<{ payload: any; exitCode: number }> { + const savedExit = process.exitCode; + const swallow = ((_chunk: unknown, ...rest: unknown[]) => { + const cb = rest.find((a) => typeof a === 'function') as (() => void) | undefined; + if (cb) cb(); + return true; + }) as typeof process.stdout.write; + const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(swallow); + vi.spyOn(process.stderr, 'write').mockImplementation(swallow); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + process.chdir(projectDir); + let exitCode: number | undefined; + try { + try { + await MigrateOrganizationOwnership.run(args, { root: CLI_ROOT }); + } catch (error) { + if (!isExitSignal(error)) throw error; + exitCode = (error as { oclif?: { exit?: number } }).oclif?.exit; + } + const payload = JSON.parse(stdout.mock.calls.map((c) => String(c[0])).join('').trim()); + return { payload, exitCode: exitCode ?? (process.exitCode as number | undefined) ?? 0 }; + } finally { + process.chdir(savedCwd); + process.exitCode = savedExit; + vi.restoreAllMocks(); + } +} + +describe.skipIf(!SQLD)('os migrate organization-ownership on a Turso REMOTE database (sqld)', () => { + beforeAll(async () => { + for (const key of OVERRIDING_ENV) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + dir = mkdtempSync(join(tmpdir(), 'os-ceremony-remote-')); + projectDir = join(dir, 'project'); + mkdirSync(projectDir, { recursive: true }); + const port = await freePort(); + url = `libsql://127.0.0.1:${port}?tls=0`; + server = spawn(SQLD!, ['--http-listen-addr', `127.0.0.1:${port}`, '--db-path', join(dir, 'sqld')], { stdio: 'ignore' }); + for (let i = 0; i < 100; i++) { + try { + await secondConnection((exec) => exec('SELECT 1')); + break; + } catch { + await new Promise((r) => setTimeout(r, 100)); + } + } + + // The environment's own boot: the composition the ceremony boots, syncing. + const stack = await bootSchemaStack({ + jsonOutput: false, + databaseUrl: url, + projectRoot: projectDir, + composeHostStack: true, + composeServedPlatform: true, + composeAuthGatedSecurity: true, + }); + await stack.shutdown(); + + // The 17.x leftovers, through a connection of their own. + await secondConnection(async (exec) => { + const tables = new Set((await exec("SELECT name FROM sqlite_master WHERE type = 'table'")).map((r) => String(r.name))); + leftovers = LEFTOVER_CANDIDATES.filter((t) => tables.has(t)); + for (const t of leftovers) { + await exec(`ALTER TABLE "${t}" ADD COLUMN "organization_id" TEXT`); + await exec(`CREATE INDEX "idx_${t}_organization_id" ON "${t}" ("organization_id")`); + } + if (leftovers.includes('sys_job')) { + await exec(`INSERT INTO "sys_job" ("id", "organization_id") VALUES ('job_v17_a', NULL), ('job_v17_b', NULL)`); + } + }); + expect(leftovers.length).toBeGreaterThan(0); + }, CASE_TIMEOUT_MS); + + afterAll(() => { + server?.kill(); + for (const key of OVERRIDING_ENV) { + if (savedEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedEnv[key]; + } + if (dir) rmSync(dir, { recursive: true, force: true }); + }); + + it('plans read-only, applies with blocking 0, and the drops land on the server', async () => { + for (const t of leftovers) expect(await hasOrganizationColumn(t)).toBe(true); + const planFile = join(dir, 'plan.json'); + + const before = await logicalDump(); + const plan = await run(['--database-url', url, '--json', '--out', planFile]); + expect(plan.payload.error, JSON.stringify(plan.payload).slice(0, 800)).toBeUndefined(); + expect(plan.exitCode).toBe(0); + expect(await logicalDump()).toBe(before); + expect(plan.payload.plan.database).toBe(`libsql://127.0.0.1:${new URL(url).port}`); + + const apply = await run([ + '--apply', '--backup-taken', '--plan-file', planFile, '--database-url', url, '--json', '--out', join(dir, 'apply.json'), + ]); + expect(apply.payload.error, JSON.stringify(apply.payload).slice(0, 800)).toBeUndefined(); + expect([apply.exitCode, apply.payload.report.complete, apply.payload.report.blocking]).toEqual([0, true, 0]); + + for (const t of leftovers) expect(await hasOrganizationColumn(t)).toBe(false); + const after = await secondConnection(async (exec) => ({ + indexes: (await exec("SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_%_organization_id'")) + .map((r) => String(r.name)) + .filter((n) => leftovers.some((t) => n === `idx_${t}_organization_id`)), + marker: (await exec( + 'SELECT applied_at IS NOT NULL AS applied, verified_at IS NOT NULL AS verified FROM sys_migration WHERE id = ?', + [ORGANIZATION_OWNERSHIP_MIGRATION_ID], + )).map((r) => ({ applied: Number(r.applied), verified: Number(r.verified) })), + jobs: leftovers.includes('sys_job') ? Number((await exec('SELECT count(*) AS n FROM "sys_job"'))[0]!.n) : 2, + })); + expect(after).toEqual({ indexes: [], marker: [{ applied: 1, verified: 1 }], jobs: 2 }); + }, CASE_TIMEOUT_MS); +}); diff --git a/packages/cli/src/utils/schema-migration-plugins.test.ts b/packages/cli/src/utils/schema-migration-plugins.test.ts index b3a290e14a5..7c067b4fcf2 100644 --- a/packages/cli/src/utils/schema-migration-plugins.test.ts +++ b/packages/cli/src/utils/schema-migration-plugins.test.ts @@ -770,6 +770,36 @@ describe('measureComposedCoverage (#13028)', () => { expect(out.notes.join(' ')).toContain('did not defer schema DDL'); expect(out.notes.join(' ')).toContain('UNMEASURED'); }); + + it('[#23816] on a boot that SKIPPED schema sync, binds each object DDL-free and never syncs', async () => { + // The Turso REMOTE primary cannot defer DDL, so the organization-ownership + // ceremony boots it with schema sync skipped. The composed host's objects + // still have to reach the driver for its drift report, and the one binding + // that runs no DDL is `registerObjectMetadata`, the engine's own skip-sync + // pass. `syncObjectSchema` would take the DDL path there. + const registered: string[] = []; + const planned = { + ...driverDouble('remote'), + registerObjectMetadata: (objects: Array<{ name: string }>) => { + for (const o of objects) registered.push(o.name); + }, + }; + const withoutRegistration = driverDouble('older'); + let synced = 0; + const engine = engineDouble({ + objects: [{ name: 'sys_position' }, { name: 'sys_job' }, { name: 'legacy_row' }], + driverFor: (n) => (n === 'legacy_row' ? withoutRegistration : planned), + sync: async () => { synced++; }, + }); + + const out = await measureComposedCoverage(kernelWith(engine), [planned, withoutRegistration], false, { ddlFree: true }); + + expect(synced).toBe(0); + expect(registered).toEqual(['sys_position', 'sys_job']); + expect(out.coverage).toMatchObject({ registeredObjects: 3, examinedObjects: 2, unexaminedObjects: 1 }); + expect(out.coverage.reasons.unsupported).toBe(1); + expect(out.notes.join(' ')).not.toContain('did not defer schema DDL'); + }); }); /** From 52965b7ccae9af525b841de08c2b9ea84d7dd2dc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 07:27:38 +0000 Subject: [PATCH 05/10] test(driver-turso): an overlay-index rebuild through the remote face's two drift members Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01Lr3rg6qTp9r38exA5DHEEH --- .../turso-remote-migration-entries.test.ts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/packages/drivers/driver-turso/src/turso-remote-migration-entries.test.ts b/packages/drivers/driver-turso/src/turso-remote-migration-entries.test.ts index da614978510..01a99bb828c 100644 --- a/packages/drivers/driver-turso/src/turso-remote-migration-entries.test.ts +++ b/packages/drivers/driver-turso/src/turso-remote-migration-entries.test.ts @@ -264,3 +264,50 @@ describe('the database a migration names', () => { expect('connection' in config).toBe(false); }); }); + +describe('an overlay-index rebuild, the step a host upgrade takes through these two members', () => { + // The shape a host's v18 upgrade drives: `detectManagedDrift([subject])` for + // one declared object, apply the one `recreate_index` it reports, and read + // again, where an entry still there means the rebuild did not take. The + // object here is test data shaped like an overlay table (a global unique over + // four columns, two of them nullable); the stale index is the 17.x shape + // under the same name. + const OVERLAY = { + name: 'overlay_t', + fields: { + type: { type: 'text' }, name: { type: 'text' }, package_id: { type: 'text' }, bundle_member: { type: 'text' }, + }, + indexes: [{ name: 'idx_overlay_t_active', fields: ['type', 'name', 'package_id', 'bundle_member'], unique: 'global' }], + }; + + it('the stale index is rebuilt to its declared shape, and neither face reports it after', async () => { + const file = nextFile(); + const remote = await remoteOver(file); + await remote.syncSchemasBatch([{ object: 'overlay_t', schema: OVERLAY }]); + const declared = await indexesOf(file, 'overlay_t'); + await onDisk( + file, + 'ALTER TABLE "overlay_t" ADD COLUMN "organization_id" TEXT', + 'DROP INDEX "idx_overlay_t_active"', + // Plain key parts: an expression key is outside what the shared differ + // reproduces (`isSyncReproducibleIndex`), on either face. + 'CREATE UNIQUE INDEX "idx_overlay_t_active" ON "overlay_t" ("type", "name", "organization_id", "package_id")', + ); + const subject = [OVERLAY] as unknown as Objects; + const find = async () => (await remote.detectManagedDrift(subject)).find( + (d) => d.op.type === 'recreate_index' && (d.op as { indexName?: string }).indexName === 'idx_overlay_t_active', + ); + const entry = await find(); + expect(entry).toBeDefined(); + + await remote.applyMigrationEntries([entry!], { allowDestructive: true }); + + expect(await find()).toBeUndefined(); + expect((await indexesOf(file, 'overlay_t')).idx_overlay_t_active).toBe(declared.idx_overlay_t_active); + const local = new TursoDriver({ url: `file:${file}` }); + open.push({ driver: local, client: null }); + await local.connect(); + local.registerObjectMetadata(subject); + expect((await local.detectManagedDrift(subject)).filter((d) => d.op.type === 'recreate_index')).toEqual([]); + }); +}); From 39b9abdda3f5d8484e6c7e164a60b743a346db46 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 07:28:27 +0000 Subject: [PATCH 06/10] chore(changeset): the v18 ceremony on a Turso remote database Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01Lr3rg6qTp9r38exA5DHEEH --- .../23816-turso-remote-ownership-ceremony.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .changeset/23816-turso-remote-ownership-ceremony.md diff --git a/.changeset/23816-turso-remote-ownership-ceremony.md b/.changeset/23816-turso-remote-ownership-ceremony.md new file mode 100644 index 00000000000..b239725cda6 --- /dev/null +++ b/.changeset/23816-turso-remote-ownership-ceremony.md @@ -0,0 +1,20 @@ +--- +'@objectstack/driver-turso': minor +'@objectstack/cli': minor +'@objectstack/runtime': minor +--- + +feat(driver-turso,cli): the v18 organization-ownership ceremony runs on a Turso remote database + +Clause-②: yes (widening) + +`os migrate organization-ownership`, the plan and `--apply --backup-taken`, now completes against a Turso database reached over the remote transport (`libsql://…`), the transport hosted environments use. Before, the plan stopped at its boot with `Deferred schema DDL is not supported by the Turso REMOTE transport`, and the apply's column drops had no remote path either. + +- **`@objectstack/driver-turso`**, remote face: + - `detectManagedDrift()` answers instead of refusing with `NOT_IMPLEMENTED` / 501. The verdict is the shared differ's, the one the local face runs; the physical schema it diffs is read from the remote catalog (`pragma_table_info`, `sqlite_master`, `pragma_index_list` / `pragma_index_info`, and the ADR-0120 D4 duplicate probe). The declarations it diffs against are recorded by every remote schema door and by the DDL-free `registerObjectMetadata`, so a boot that skipped schema sync is covered too. + - `applyMigrationEntries()` applies `drop_index`, `create_index`, `recreate_index`, `replace_unique_index` and `drop_column` instead of refusing. Each entry runs as one libSQL `write` batch, so it lands whole or not at all: a `drop_column` drops the column's indexes with it in the same transaction, and a re-created index that fails to build leaves the old one in place. An entry is reported `applied` only once a read-back shows its target state. Every other column entry (nullability, width, a dropped DEFAULT) needs the SQLite table rebuild, which this transport does not run, so it is reported `skipped` and logged, as the in-place dialects report an op they cannot perform. + - The remote driver's `config` now carries its `url`, so the CLI labels a remote database by its URL (credential-free) instead of `better-sqlite3`. The ceremony writes that label into its plan and refuses an apply whose plan names another database; that refusal can now tell two remote databases apart. +- **`@objectstack/cli`**: the ceremony's plan and apply boots skip schema sync, instead of arming deferred DDL, when the primary datasource is a Turso remote one. No DDL is attempted, so the plan stays read-only, and every other datasource the boot connects is still deferred. The environment's own boot has already run the additive sync the ceremony reads. Every other `os migrate` command keeps refusing on that transport: their dry runs list held-back schema work, which a boot that holds nothing back cannot list. +- **`@objectstack/runtime`**: `createStandaloneStack` takes `skipSchemaSync`, passed to `ObjectQLPlugin` as the option `OS_SKIP_SCHEMA_SYNC=1` names. Unset, the plugin reads the variable as before. + +One behaviour change outside the ceremony: on an `os serve` boot pinned with `OS_ARTIFACT_URL`, the boot's migration gate now judges a Turso remote database as it judges every other one. It applies safe drift and refuses the boot on destructive drift. Until now it printed that it could not check the schema and booted. From ec063d8f3d85acb4e7dbb82a8c98a2fc6057ec5e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 08:38:52 +0000 Subject: [PATCH 07/10] fix(driver-turso,cli): the OS_ARTIFACT_URL boot gate keeps its warn-and-boot answer on a Turso remote primary The remote face's schema sync creates no declared NOT NULL, so a healthy remote database reads as destructive tighten_not_null, and the gate's remedy cannot run on that transport. TursoDriver declares driftReportCaveat (a sentence on the remote face, null elsewhere) and the gate, reading that declaration, warns and boots, applying and refusing nothing. Also from the contract review: the embedded-replica drift control is back beside the remote suite; the remote detector reads its catalog in three round trips per call; catalog reads refuse a non-row answer; a created index counts as applied only with the uniqueness and key columns asked for; the parity suite is cited by its real name. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01Lr3rg6qTp9r38exA5DHEEH --- .../src/utils/artifact-boot-migration.test.ts | 34 +++ .../cli/src/utils/artifact-boot-migration.ts | 17 ++ packages/cli/src/utils/schema-migrate.ts | 7 + .../drivers/driver-turso/src/turso-driver.ts | 281 ++++++++++++++---- .../src/turso-remote-drift-detection.test.ts | 88 ++++++ 5 files changed, 368 insertions(+), 59 deletions(-) diff --git a/packages/cli/src/utils/artifact-boot-migration.test.ts b/packages/cli/src/utils/artifact-boot-migration.test.ts index 27fdb9e73bb..4f5eac0073f 100644 --- a/packages/cli/src/utils/artifact-boot-migration.test.ts +++ b/packages/cli/src/utils/artifact-boot-migration.test.ts @@ -214,3 +214,37 @@ describe('formatDestructiveDriftRefusal', () => { expect(text).toContain('never skipped in silence'); }); }); + +describe('runArtifactBootMigrationGate — a driver that declares its report must not be acted on', () => { + // [#23816] The Turso REMOTE face declares `driftReportCaveat`: its schema + // sync does not create a declared NOT NULL, so a healthy remote database + // reports destructive `tighten_not_null`, and the refusal's remedy cannot + // run on that transport. The gate keeps its "did not run" answer there. + const CAVEAT = 'the transport cannot act on its own report'; + + it('warns with the reason, applies nothing and refuses nothing, and never reads the report', async () => { + const { driver, applyCalls } = fakeDriver([entry('destructive'), entry('safe')]); + const declared: SqlDriverLike = { ...driver, driftReportCaveat: CAVEAT }; + const warned: string[] = []; + + const verdict = await runArtifactBootMigrationGate({ + driver: declared, artifactDisplay: ARTIFACT, warn: (m) => warned.push(m), + }); + + expect(verdict).toEqual({ ok: true, applied: [], skipped: [], destructive: [] }); + expect(driver.detectManagedDrift).not.toHaveBeenCalled(); + expect(applyCalls).toHaveLength(0); + expect(warned).toHaveLength(1); + expect(warned[0]).toContain(CAVEAT); + expect(warned[0]).toContain('Boot continues'); + }); + + it('the control: a null caveat is no caveat, and the same report refuses the boot', async () => { + const { driver } = fakeDriver([entry('destructive')]); + const verdict = await runArtifactBootMigrationGate({ + driver: { ...driver, driftReportCaveat: null }, artifactDisplay: ARTIFACT, + }); + expect(verdict.ok).toBe(false); + expect(driver.detectManagedDrift).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/cli/src/utils/artifact-boot-migration.ts b/packages/cli/src/utils/artifact-boot-migration.ts index 6a2302f3adb..d42cc2ed4a1 100644 --- a/packages/cli/src/utils/artifact-boot-migration.ts +++ b/packages/cli/src/utils/artifact-boot-migration.ts @@ -25,6 +25,11 @@ * - **nothing is skipped silently** — the third state, "shrug and serve", is * the one this gate exists to delete. * + * One driver-declared exception, and it is loud: a driver whose + * `driftReportCaveat` says its report must not be acted on unattended (the + * Turso REMOTE face, #23816) gets a warning naming that reason, and the gate + * applies and refuses nothing — as when the report cannot be read at all. + * * `needs_confirm` drift is applied along with `safe`: `os migrate apply`'s own * split is `category !== 'destructive' || allowDestructive`, so "the * `--allow-destructive` class" the acceptance names is exactly @@ -120,6 +125,18 @@ export async function runArtifactBootMigrationGate(opts: { // no drift to classify and nothing for this policy to decide. if (!driver) return { ok: true, applied: [], skipped: [], destructive: [] }; + // [#23816] A driver that declares its report must not be acted on + // unattended gets this gate's "did not run" answer: warn, apply nothing, + // refuse nothing. The Turso REMOTE face declares one: its schema sync does + // not create a declared NOT NULL, so a healthy remote database reads as + // destructive drift, and the refusal's remedy cannot run on that transport. + // Read from the driver's declaration, never from its transport name. + const caveat = driver.driftReportCaveat; + if (typeof caveat === 'string' && caveat) { + warn(` ⚠ Did not check the physical schema against the artifact: ${caveat}. Boot continues.`); + return { ok: true, applied: [], skipped: [], destructive: [] }; + } + let drift: ManagedDriftEntry[]; try { drift = await driver.detectManagedDrift(); diff --git a/packages/cli/src/utils/schema-migrate.ts b/packages/cli/src/utils/schema-migrate.ts index 651b53dca0c..3c251f3b490 100644 --- a/packages/cli/src/utils/schema-migrate.ts +++ b/packages/cli/src/utils/schema-migrate.ts @@ -80,6 +80,13 @@ export interface SqlDriverLike { * caller's own refusal branch can tell an operator which it was. */ planMediaColumnMove?: () => Promise; + /** + * [#23816] Why this driver's drift report must not be acted on unattended, + * or `null` / absent when it may be — `TursoDriver`'s REMOTE face declares + * one. Read by the artifact-pinned boot gate (`./artifact-boot-migration.ts`), + * which then neither applies nor refuses on the report. + */ + readonly driftReportCaveat?: string | null; /** Deferred-DDL surface (#3917) — optional, so a driver without it still boots. */ setDeferredDdl?: (deferred: boolean) => void; previewDeferredSchemaWork?: () => Promise; diff --git a/packages/drivers/driver-turso/src/turso-driver.ts b/packages/drivers/driver-turso/src/turso-driver.ts index 1316b9f8ee2..efe4a3b627b 100644 --- a/packages/drivers/driver-turso/src/turso-driver.ts +++ b/packages/drivers/driver-turso/src/turso-driver.ts @@ -579,8 +579,11 @@ function refuseRemoteDeferredDdl(): never { * The declared side is {@link TursoDriver.remoteDeclaredSchemas}: what each * object that reached this face declares, recorded by every remote schema door * and by the DDL-free registration a `skipSchemaSync` boot takes. - * `turso-local-remote-drift-parity.test.ts` holds the two faces to one answer + * `turso-remote-drift-detection.test.ts` holds the two faces to one answer * over one SQLite file, both for a clean table and for each drifted shape. + * The no-argument call reads the whole catalog it needs in three round trips, + * whatever the number of tables (`readRemoteCatalog`), because a hosted + * tenant's first v18 request pays for this read. * * # What the apply door answers here * @@ -593,9 +596,31 @@ function refuseRemoteDeferredDdl(): never { * DEFAULT) needs that rebuild, which this transport has no copy of, so it is * reported `skipped` and logged, as the in-place arm reports an op its dialect * cannot perform. Each entry is reported `applied` only once a read-back shows - * the physical schema in its target state. + * the physical schema in its target state: the column gone, the index name + * gone, or the index present with the uniqueness and the key columns, in + * order, the entry asked for. + * + * # What this face does NOT claim: a boot may not act on its report + * + * The report is accurate about the database, but not every finding is real + * drift. This face's schema sync (`RemoteTransport`) does not create the NOT NULL + * constraint a field declares, so a table it created reads as `tighten_not_null` + * (destructive) against its own declaration. Nor is there a remedy for a + * destructive entry on this transport: `os migrate apply` arms deferred DDL, + * which this face refuses. So {@link TursoDriver.driftReportCaveat} says so, + * and a caller that would apply or refuse on the report unattended (the + * `OS_ARTIFACT_URL` boot gate) reads the caveat and does neither. The ceremony + * is not such a caller: it applies only the column and index drops it named. */ +/** [#23816] {@link TursoDriver.driftReportCaveat} on the REMOTE face. */ +const REMOTE_DRIFT_REPORT_CAVEAT = + 'the Turso REMOTE transport\'s schema sync does not create the NOT NULL constraint a field declares, so ' + + 'this driver\'s drift report can name a tightening its own sync produced, and a destructive entry has no ' + + 'remedy on this transport (`os migrate apply` refuses against it). The report is for reading; nothing is ' + + 'applied or refused on it unattended. To review the drift, run `os migrate plan` against a local SQLite ' + + 'copy of this database (a `file:` URL)'; + /** * One `pragma_table_info` row as Knex's SQLite `columnInfo()` reports it — the * shape `SqlDriver.introspectColumns` reads its per-column facts from: the @@ -620,6 +645,53 @@ function remoteIntrospectedColumn(row: Record): IntrospectedCol /** A SQLite identifier, quoted, with any embedded quote doubled. */ const quoteRemoteIdentifier = (name: string): string => `"${String(name).replace(/"/g, '""')}"`; +/** [#23816] The catalog statements the remote drift reads send, one spelling each. */ +const REMOTE_TABLE_INFO_SQL = 'SELECT cid, name, type, "notnull", dflt_value FROM pragma_table_info(?) ORDER BY cid'; +const REMOTE_INDEX_LIST_SQL = 'SELECT name, "unique", origin, partial FROM pragma_index_list(?) ORDER BY seq'; +const REMOTE_INDEX_INFO_SQL = 'SELECT name FROM pragma_index_info(?) ORDER BY seqno'; +/** [#23816] Statements per catalog `read` batch: a bound on one request, not a tuning knob. */ +const REMOTE_CATALOG_BATCH = 200; + +/** [#23816] What one remote drift detection reads, keyed by table and by index name. */ +interface RemoteCatalogSnapshot { + /** `pragma_table_info` rows per table the database holds. */ + columns: Map>>; + /** `pragma_index_list` rows per table the database holds. */ + indexList: Map>>; + /** Each index's DDL, from `sqlite_master`, where SQLite keeps one. */ + indexDdl: Map; + /** `pragma_index_info` rows per index whose DDL does not parse. */ + indexInfo: Map>>; +} + +/** + * [#23816] A table's indexes from its `pragma_index_list` rows, built as the + * local SQLite arm of `SqlDriver.introspectIndexes` builds them: the key from + * the parsed DDL where it parses (an expression key or a predicate survives + * only there), else from `pragma_index_info`. + */ +function remotePhysicalIndexes( + list: ReadonlyArray>, + ddlOf: (name: string) => string | undefined, + infoOf: (name: string) => ReadonlyArray>, +): PhysicalIndex[] { + const byName = new Map(); + for (const r of list) { + const name = String(r.name); + let entry = byName.get(name); + if (!entry) byName.set(name, (entry = { name, columns: [], unique: Number(r.unique) === 1, primary: r.origin === 'pk' })); + const parsed = parseIndexDdl(ddlOf(name) ?? ''); + if (parsed) { + applyIndexKeyParts(entry, parsed.keyParts); + if (parsed.partial) entry.partial = true; + } else { + for (const c of infoOf(name)) if (c.name != null) entry.columns.push(String(c.name)); + } + if (Number(r.partial) === 1) entry.partial = true; + } + return [...byName.values()]; +} + // ── Remote media column move planning: refused, never "nothing to move" ────── /** @@ -3720,14 +3792,28 @@ export class TursoDriver extends SqlDriver { super.setDeferredDdl(deferred); } + /** + * Why this driver's drift report must not be ACTED ON unattended, or `null` + * when it may be. [#23816] A sentence on the REMOTE face (see "Remote schema + * drift" at the top of this file); `null` on the local and embedded-replica + * faces, whose report is the Knex detector's. A caller that would apply or + * refuse on the report with nobody watching (the `OS_ARTIFACT_URL` boot gate) + * reads this first; a caller that applies only the entries it named itself + * (the organization-ownership ceremony's column drops) does not need to. + */ + get driftReportCaveat(): string | null { + return this.isRemote ? REMOTE_DRIFT_REPORT_CAVEAT : null; + } + /** * Detect managed-schema drift. [#23816] On the REMOTE face: the shared * differ, fed by the remote catalog (see "Remote schema drift" at the top of * this file). With no `objects` it walks {@link remoteDeclaredSchemas}, which * every remote schema door and the DDL-free {@link registerObjectMetadata} * fill, as the local face walks `managedObjectFields`. A table the database - * does not hold is skipped, as the local face skips it. Local and replica - * modes inherit the Knex detector unchanged. + * does not hold is skipped, as the local face skips it. The catalog is read + * once for the call ({@link readRemoteCatalog}), not per table. Local and + * replica modes inherit the Knex detector unchanged. * * The parameter repeats the base's declared shape key for key rather than * deriving it (`check:object-def-param-keys` arm C), so the keys a caller may @@ -3745,22 +3831,72 @@ export class TursoDriver extends SqlDriver { } else { for (const [table, declared] of this.remoteDeclaredSchemas) tables.set(table, declared); } - const present = new Set( - (await this.remoteCatalogRead(`SELECT name FROM sqlite_master WHERE type = 'table'`)).map((r) => String(r.name)), - ); - const out: ManagedDriftEntry[] = []; - for (const [table, declared] of tables) { - if (!present.has(table)) continue; - out.push(...(await this.detectTableDrift(table, declared.fields, declared.indexes))); + const catalog = await this.readRemoteCatalog([...tables.keys()]); + const previous = this.remoteCatalogSnapshot; + this.remoteCatalogSnapshot = catalog; + try { + const out: ManagedDriftEntry[] = []; + for (const [table, declared] of tables) { + if (!catalog.columns.has(table)) continue; // not a table this database holds + out.push(...(await this.detectTableDrift(table, declared.fields, declared.indexes))); + } + // The base detector's order: by table, then by column. + out.sort((a, b) => (a.table === b.table ? (a.column ?? '').localeCompare(b.column ?? '') : a.table.localeCompare(b.table))); + return out; + } finally { + if (this.remoteCatalogSnapshot === catalog) this.remoteCatalogSnapshot = previous; } - // The base detector's order: by table, then by column. - out.sort((a, b) => (a.table === b.table ? (a.column ?? '').localeCompare(b.column ?? '') : a.table.localeCompare(b.table))); - return out; + } + + /** + * [#23816] The catalog one {@link detectManagedDrift} call diffs, while it + * runs, so {@link introspectColumns} and {@link introspectIndexes} answer from + * it instead of a round trip per read. `null` outside a detection. The apply + * door never reads it: its read-backs are live ({@link remoteLiveColumns}, + * {@link remoteLiveIndexes}). + */ + private remoteCatalogSnapshot: RemoteCatalogSnapshot | null = null; + + /** + * [#23816] Read, for `tables`, everything the differ reads, in three round + * trips whatever the number of tables: `sqlite_master` (which tables exist, + * and each index's DDL); one `read` batch of `pragma_table_info` and + * `pragma_index_list` per existing table; one `read` batch of + * `pragma_index_info` for each index whose DDL does not parse (the + * autoindexes SQLite keeps no DDL for among them). A table the database does + * not hold has no `columns` entry. + */ + private async readRemoteCatalog(tables: readonly string[]): Promise { + const catalog: RemoteCatalogSnapshot = { columns: new Map(), indexList: new Map(), indexDdl: new Map(), indexInfo: new Map() }; + const present = new Set(); + for (const r of await this.remoteCatalogRead(`SELECT type, name, tbl_name, sql FROM sqlite_master WHERE type IN ('table', 'index')`)) { + if (r.type === 'table') present.add(String(r.name)); + else if (typeof r.sql === 'string' && r.sql) catalog.indexDdl.set(String(r.name), r.sql); + } + const existing = tables.filter((t) => present.has(t)); + const reads: Array<{ sql: string; args: unknown[] }> = []; + for (const t of existing) { + reads.push({ sql: REMOTE_TABLE_INFO_SQL, args: [t] }, { sql: REMOTE_INDEX_LIST_SQL, args: [t] }); + } + const answers = await this.remoteCatalogBatch(reads); + existing.forEach((t, i) => { + catalog.columns.set(t, answers[2 * i]!); + catalog.indexList.set(t, answers[2 * i + 1]!); + }); + const needInfo = [...catalog.indexList.values()] + .flat() + .map((r) => String(r.name)) + .filter((name) => !parseIndexDdl(catalog.indexDdl.get(name) ?? '')); + const infos = await this.remoteCatalogBatch(needInfo.map((name) => ({ sql: REMOTE_INDEX_INFO_SQL, args: [name] }))); + needInfo.forEach((name, i) => catalog.indexInfo.set(name, infos[i]!)); + return catalog; } /** * [#23816] One catalog read on the REMOTE database, in the raw-statement - * fault envelope `execute()` answers with. + * fault envelope `execute()` answers with. An answer that is not a row list + * is a fault too, never read as an empty catalog: an empty catalog is "no + * drift". */ private async remoteCatalogRead(sql: string, args: unknown[] = []): Promise>> { let rows: unknown; @@ -3769,73 +3905,91 @@ export class TursoDriver extends SqlDriver { } catch (error) { throw this.rawStatementFault(sql, error); } - return (Array.isArray(rows) ? rows : []) as Array>; + if (!Array.isArray(rows)) throw this.rawStatementFault(sql, new Error('the remote database answered no row list')); + return rows as Array>; + } + + /** [#23816] {@link remoteCatalogRead} for several statements, as one `read` batch (chunked). */ + private async remoteCatalogBatch(reads: ReadonlyArray<{ sql: string; args: unknown[] }>): Promise>>> { + const out: Array>> = []; + for (let i = 0; i < reads.length; i += REMOTE_CATALOG_BATCH) { + const chunk = reads.slice(i, i + REMOTE_CATALOG_BATCH); + let answers: unknown[]; + try { + answers = await this.remoteTransport!.batch(chunk.map((r) => ({ sql: r.sql, args: r.args as never })), 'read'); + } catch (error) { + throw this.rawStatementFault(chunk[0]!.sql, error); + } + chunk.forEach((r, j) => { + if (!Array.isArray(answers[j])) throw this.rawStatementFault(r.sql, new Error('the remote database answered no row list')); + out.push(answers[j] as Array>); + }); + } + return out; } /** - * The table's columns, in declared order. [#23816] On the REMOTE face, read - * from `pragma_table_info` and shaped as Knex's SQLite `columnInfo()` shapes - * it ({@link remoteIntrospectedColumn}); the cid is the ordinal, as + * The table's columns, in declared order. [#23816] On the REMOTE face, from + * `pragma_table_info` (a detection's catalog while one runs, else a live + * read), shaped as Knex's SQLite `columnInfo()` shapes it + * ({@link remoteIntrospectedColumn}); the cid is the ordinal, as * `introspectColumnOrder` reads it locally. */ protected override async introspectColumns(tableName: string): Promise { if (!this.isRemote) return super.introspectColumns(tableName); - const rows = await this.remoteCatalogRead( - 'SELECT cid, name, type, "notnull", dflt_value FROM pragma_table_info(?) ORDER BY cid', - [tableName], - ); - return rows.map(remoteIntrospectedColumn); + const held = this.remoteCatalogSnapshot?.columns.get(tableName); + return held ? held.map(remoteIntrospectedColumn) : this.remoteLiveColumns(tableName); + } + + /** [#23816] The table's columns, read live from the remote catalog. */ + private async remoteLiveColumns(tableName: string): Promise { + return (await this.remoteCatalogRead(REMOTE_TABLE_INFO_SQL, [tableName])).map(remoteIntrospectedColumn); } /** * The table's indexes. [#23816] On the REMOTE face, the statements the local - * SQLite arm sends, sent to the remote database: `sqlite_master` for each - * index's DDL (the only place an expression key or a `WHERE` predicate - * survives), `pragma_index_list` for the set, and `pragma_index_info` for an - * index whose DDL does not parse. The DDL goes through the driver-sql - * parsers the local arm uses, so an index reads alike on both faces. + * SQLite arm sends, sent to the remote database (a detection's catalog while + * one runs, else live reads): `sqlite_master` for each index's DDL (the only + * place an expression key or a `WHERE` predicate survives), + * `pragma_index_list` for the set, and `pragma_index_info` for an index whose + * DDL does not parse. The DDL goes through the driver-sql parsers the local + * arm uses, so an index reads alike on both faces. */ protected override async introspectIndexes( tableName: string, opts: { onFailure?: 'throw' | 'partial' } = {}, ): Promise { if (!this.isRemote) return super.introspectIndexes(tableName, opts); - const byName = new Map(); - const upsert = (name: string, unique: boolean, primary: boolean): PhysicalIndex => { - let entry = byName.get(name); - if (!entry) byName.set(name, (entry = { name, columns: [], unique, primary })); - return entry; - }; + const held = this.remoteCatalogSnapshot; + const list = held?.indexList.get(tableName); + if (held && list) { + return remotePhysicalIndexes(list, (name) => held.indexDdl.get(name), (name) => held.indexInfo.get(name) ?? []); + } + return this.remoteLiveIndexes(tableName, opts); + } + + /** [#23816] The table's indexes, read live from the remote catalog. */ + private async remoteLiveIndexes(tableName: string, opts: { onFailure?: 'throw' | 'partial' } = {}): Promise { + const ddlByName = new Map(); + const info = new Map>>(); + let list: Array> = []; try { - const ddlByName = new Map(); for (const r of await this.remoteCatalogRead( `SELECT name, sql FROM sqlite_master WHERE type = 'index' AND tbl_name = ?`, [tableName], )) { if (typeof r.sql === 'string' && r.sql) ddlByName.set(String(r.name), r.sql); } - for (const r of await this.remoteCatalogRead( - 'SELECT name, "unique", origin, partial FROM pragma_index_list(?) ORDER BY seq', - [tableName], - )) { + list = await this.remoteCatalogRead(REMOTE_INDEX_LIST_SQL, [tableName]); + for (const r of list) { const name = String(r.name); - const entry = upsert(name, Number(r.unique) === 1, r.origin === 'pk'); - const parsed = parseIndexDdl(ddlByName.get(name) ?? ''); - if (parsed) { - applyIndexKeyParts(entry, parsed.keyParts); - if (parsed.partial) entry.partial = true; - } else { - for (const c of await this.remoteCatalogRead('SELECT name FROM pragma_index_info(?) ORDER BY seqno', [name])) { - if (c.name != null) entry.columns.push(String(c.name)); - } - } - if (Number(r.partial) === 1) entry.partial = true; + if (!parseIndexDdl(ddlByName.get(name) ?? '')) info.set(name, await this.remoteCatalogRead(REMOTE_INDEX_INFO_SQL, [name])); } } catch (e) { // Only a caller that can CORRECT a short read may ask for one (#7332). if (opts.onFailure !== 'partial') throw e; } - return [...byName.values()]; + return remotePhysicalIndexes(list, (name) => ddlByName.get(name), (name) => info.get(name) ?? []); } /** @@ -4323,9 +4477,9 @@ export class TursoDriver extends SqlDriver { ); return false; } - const hasColumn = async () => (await this.introspectColumns(op.table)).some((c) => c.name === column); + const hasColumn = async () => (await this.remoteLiveColumns(op.table)).some((c) => c.name === column); if (!(await hasColumn())) return false; - const keyed = (await this.introspectIndexes(op.table)).filter( + const keyed = (await this.remoteLiveIndexes(op.table)).filter( (i) => !i.primary && !i.name.startsWith('sqlite_autoindex_') && i.columns.includes(column), ); await this.remoteTransport!.batch( @@ -4344,9 +4498,18 @@ export class TursoDriver extends SqlDriver { * one `write` batch, so a create that fails (duplicates under a new UNIQUE, * a column that is not there) leaves the old index exactly as it was. The * `CREATE INDEX` is the schema sync's own ({@link RemoteTransport.createIndexSql}). + * A created index counts as applied only when a live read finds it with the + * uniqueness and the key columns, in order, the entry asked for; a name + * alone would also match a stale index the `IF NOT EXISTS` left in place. */ private async applyRemoteIndexDriftOp(op: ManagedDriftEntry['op']): Promise { - const names = async () => new Set((await this.introspectIndexes(op.table)).map((i) => i.name)); + const live = () => this.remoteLiveIndexes(op.table); + const names = async () => new Set((await live()).map((i) => i.name)); + const built = async (name: string, columns: readonly string[], unique: boolean) => { + const found = (await live()).find((i) => i.name === name); + return !!found && found.unique === unique && found.columns.length === columns.length + && found.columns.every((c, i) => c === columns[i]); + }; const create = (name: string, columns: string[], unique: boolean, nullSafeColumns?: string[]) => this.remoteTransport!.createIndexSql(op.table, { name, columns, unique, ...(nullSafeColumns ? { nullSafeColumns } : {}) }); const drop = (name: string) => `DROP INDEX IF EXISTS ${quoteRemoteIdentifier(name)}`; @@ -4359,7 +4522,7 @@ export class TursoDriver extends SqlDriver { case 'create_index': { if (!(await this.remoteIndexColumnsPresent(op.table, op.indexName, op.columns))) return false; await this.remoteTransport!.batch([create(op.indexName, op.columns, op.unique, op.nullSafeColumns)], 'write'); - return (await names()).has(op.indexName); + return built(op.indexName, op.columns, op.unique); } case 'recreate_index': { // ADR-0120 D4: the NULL-safe tightening re-runs the duplicate pre-flight @@ -4380,7 +4543,7 @@ export class TursoDriver extends SqlDriver { [drop(op.indexName), create(op.indexName, op.columns, op.unique, op.nullSafeColumns)], 'write', ); - return (await names()).has(op.indexName); + return built(op.indexName, op.columns, op.unique); } case 'replace_unique_index': { // CREATE before DROP, in one transaction: uniqueness is never @@ -4391,7 +4554,7 @@ export class TursoDriver extends SqlDriver { 'write', ); const after = await names(); - return after.has(op.createIndexName) && op.dropIndexNames.every((n) => !after.has(n)); + return (await built(op.createIndexName, op.createColumns, true)) && op.dropIndexNames.every((n) => !after.has(n)); } default: // `unbuildable_index` is report-only: no column to build it over. @@ -4406,7 +4569,7 @@ export class TursoDriver extends SqlDriver { * there and here: the index the metadata declares is not created. */ private async remoteIndexColumnsPresent(table: string, indexName: string, columns: readonly string[]): Promise { - const physical = new Set((await this.introspectColumns(table)).map((c) => c.name)); + const physical = new Set((await this.remoteLiveColumns(table)).map((c) => c.name)); const missing = columns.filter((c) => !physical.has(c)); if (missing.length === 0) return true; this.logDurabilityFailure( diff --git a/packages/drivers/driver-turso/src/turso-remote-drift-detection.test.ts b/packages/drivers/driver-turso/src/turso-remote-drift-detection.test.ts index f4ff51d00d8..fe13d49705a 100644 --- a/packages/drivers/driver-turso/src/turso-remote-drift-detection.test.ts +++ b/packages/drivers/driver-turso/src/turso-remote-drift-detection.test.ts @@ -33,6 +33,14 @@ * 3. The DDL-free `registerObjectMetadata` (a `skipSchemaSync` boot) fills the * registry the no-argument call walks, and sends nothing to the database. * 4. A declared table the database does not hold is skipped, as locally. + * 5. The embedded-replica face, which reads its local file through Knex, still + * reports drift on a drifted database (the control the retired refusal + * suite carried for this face, kept here at the same layer). + * 6. `driftReportCaveat`: a sentence on the remote face, which a caller that + * would act on the report unattended reads first; `null` on the local and + * embedded-replica faces. + * 7. The no-argument call reads the catalog in the same number of round trips + * for one table as for many. */ import { mkdtempSync, rmSync } from 'node:fs'; @@ -41,6 +49,8 @@ import { join } from 'node:path'; import { afterAll, afterEach, describe, expect, it } from 'vitest'; import { createClient, type Client } from '@libsql/client'; import { TursoDriver } from './turso-driver.js'; +import { makeLibsqlSqliteStub } from './libsql-sqlite-stub.testkit.js'; +import { replicaFiles } from './replica-file.testkit.js'; const scratch = mkdtempSync(join(tmpdir(), 'turso-remote-drift-')); afterAll(() => rmSync(scratch, { recursive: true, force: true })); @@ -220,3 +230,81 @@ describe('the registry the no-argument call walks', () => { expect(await remote.detectManagedDrift([T, absent])).toEqual([]); }); }); + +describe('the embedded-replica face still reports drift (its Knex detector)', () => { + const replicas = replicaFiles(); + afterAll(() => replicas.removeAll()); + + it.each(['no arguments', 'explicit objects'] as const)('%s', async (call) => { + // Replica mode reads its local file through Knex; `sync.onConnect: false` + // keeps the (stubbed) sync target out of the measurement. + const stub = makeLibsqlSqliteStub(); + const driver = new TursoDriver({ + url: replicas.next(), + syncUrl: 'libsql://drift.example.turso.io', + client: stub as never, + sync: { onConnect: false }, + }); + open.push({ driver, client: null }); + await driver.connect(); + expect(driver.transportMode).toBe('replica'); + await driver.initObjects([T] as never); + await driver.execute('ALTER TABLE "t" ADD COLUMN "legacy" TEXT'); + + const drift = call === 'no arguments' ? await driver.detectManagedDrift() : await driver.detectManagedDrift([T]); + + expect(drift).toEqual([expect.objectContaining({ + table: 't', column: 'legacy', kind: 'unmapped_column', category: 'destructive', op: expect.objectContaining({ type: 'drop_column' }), + })]); + stub.close(); + }); +}); + +describe('driftReportCaveat: whether a caller may act on the report unattended', () => { + it('a sentence on the remote face; null on the local and embedded-replica faces', async () => { + const { driver: remote } = await remoteOver(nextFile()); + expect(remote.driftReportCaveat).toEqual(expect.stringContaining('NOT NULL')); + + const local = new TursoDriver({ url: `file:${nextFile()}` }); + expect(local.transportMode).toBe('local'); + expect(local.driftReportCaveat).toBeNull(); + + const replicas = replicaFiles(); + try { + const replica = new TursoDriver({ url: replicas.next(), syncUrl: 'libsql://drift.example.turso.io', sync: { onConnect: false } }); + expect(replica.transportMode).toBe('replica'); + expect(replica.driftReportCaveat).toBeNull(); + } finally { + replicas.removeAll(); + } + }); +}); + +describe('the no-argument call reads the catalog in a fixed number of round trips', () => { + /** Every client call a detection makes, counted, one batch as one call. */ + async function roundTripsFor(objects: Objects): Promise { + const file = nextFile(); + const { driver: synced } = await remoteOver(file); + await SYNC.syncSchemasBatch(synced, objects); + const client = createClient({ url: `file:${file}` }); + let calls = 0; + const counting = { + execute: (stmt: unknown) => { calls++; return client.execute(stmt as never); }, + batch: (stmts: unknown[], mode?: unknown) => { calls++; return client.batch(stmts as never, mode as never); }, + close: () => client.close(), + } as unknown as Client; + const driver = new TursoDriver({ url: 'libsql://drift.example.turso.io', client: counting }); + open.push({ driver, client }); + await driver.connect(); + driver.registerObjectMetadata(objects); + const before = calls; + expect(await driver.detectManagedDrift()).toEqual([]); + return calls - before; + } + + it('one table and five tables cost the same three reads', async () => { + const many = ['a', 'b', 'c', 'd', 'e'].map((n) => ({ name: `rt_${n}`, fields: { name: { type: 'text' } }, indexes: [{ fields: ['name'] }] })); + expect(await roundTripsFor(many.slice(0, 1))).toBe(3); + expect(await roundTripsFor(many)).toBe(3); + }); +}); From 040fbcc89035c319e9d0da0a00c9c41b4db93dba Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 08:41:45 +0000 Subject: [PATCH 08/10] =?UTF-8?q?test(cli):=20pin=20the=20remote=20posture?= =?UTF-8?q?=20without=20a=20database=20=E2=80=94=20the=20ceremony's=20two?= =?UTF-8?q?=20opt-ins,=20transportRefusesDeferredDdl=20and=20the=20primary?= =?UTF-8?q?=20exemption?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01Lr3rg6qTp9r38exA5DHEEH --- ...anization-ownership.remote-posture.test.ts | 142 ++++++++++++++++++ .../schema-migrate.remote-posture.test.ts | 133 ++++++++++++++++ packages/cli/src/utils/schema-migrate.ts | 8 +- 3 files changed, 281 insertions(+), 2 deletions(-) create mode 100644 packages/cli/src/commands/migrate/organization-ownership.remote-posture.test.ts create mode 100644 packages/cli/src/utils/schema-migrate.remote-posture.test.ts diff --git a/packages/cli/src/commands/migrate/organization-ownership.remote-posture.test.ts b/packages/cli/src/commands/migrate/organization-ownership.remote-posture.test.ts new file mode 100644 index 00000000000..fba272b1e5d --- /dev/null +++ b/packages/cli/src/commands/migrate/organization-ownership.remote-posture.test.ts @@ -0,0 +1,142 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #23816 — `os migrate organization-ownership` declares, at both of its boots, + * that a datasource whose transport cannot defer DDL (the Turso REMOTE + * transport) boots with schema sync skipped instead + * (`deferredDdlUnsupported: 'skip-schema-sync'`), and the apply reads the + * ledger's presence from the catalog when its boot measured no table absent. + * + * Pinned on the command's own boot call, with `bootSchemaStack` stubbed: the + * posture itself is `schema-migrate.remote-posture.test.ts`'s, and the + * end-to-end run on a real `sqld` is + * `organization-ownership.remote-face.integration.test.ts` (opt-in, through + * `OS_TEST_SQLD_BIN`). Without these two opt-ins the plan stops at its boot on + * every hosted tenant again, and nothing else in CI would say so. + */ + +import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const boot = vi.hoisted(() => vi.fn(async (_opts: Record): Promise => { + throw new Error('the database was opened'); +})); +vi.mock('../../utils/schema-migrate.js', () => ({ bootSchemaStack: boot })); +// The occupancy gate is the siblings' stub (`organization-ownership.apply-flags.test.ts`). +vi.mock('../../utils/migrate-occupancy-gate.js', () => ({ + OCCUPANCY_STOP_HINT: 'occupancy hint', + probeMigrationTarget: vi.fn(async () => ({ status: 'free' })), +})); + +type Command = (typeof import('./organization-ownership.js'))['default']; +let MigrateOrganizationOwnership: Command; + +beforeAll(async () => { + ({ default: MigrateOrganizationOwnership } = await import('./organization-ownership.js')); +}, 60_000); + +const CASE_TIMEOUT_MS = 60_000; +const CLI_ROOT = fileURLToPath(new URL('../../..', import.meta.url)); + +const PLAN_DIR = mkdtempSync(join(tmpdir(), 'os-23816-posture-')); +const PLAN_FILE = join(PLAN_DIR, 'plan.json'); +writeFileSync(PLAN_FILE, JSON.stringify({ + kind: 'organization-ownership-plan', + ceremony: 'ADR-0131 D10', + generatedAt: '2026-10-16T00:00:00Z', + database: 'libsql://tenant.example.turso.io', + inventoryDigest: 'f'.repeat(64), + posture: 'single', +})); + +async function runJson(argv: string[]): Promise<{ payload: Record; exitCode: unknown }> { + const chunks: string[] = []; + const savedExit = process.exitCode; + vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown, ...rest: unknown[]) => { + chunks.push(String(chunk)); + const cb = rest.find((a) => typeof a === 'function') as (() => void) | undefined; + if (cb) cb(); + return true; + }) as typeof process.stdout.write); + vi.spyOn(process.stderr, 'write').mockImplementation((() => true) as typeof process.stderr.write); + try { + await MigrateOrganizationOwnership.run(argv, { root: CLI_ROOT }); + } catch (error) { + if ((error as { oclif?: unknown }).oclif === undefined) throw error; + } + const exitCode = process.exitCode; + process.exitCode = savedExit; + vi.restoreAllMocks(); + return { payload: JSON.parse(chunks.join('').trim()) as Record, exitCode }; +} + +afterEach(() => { + boot.mockReset(); + boot.mockImplementation(async () => { + throw new Error('the database was opened'); + }); +}); + +const POSTURE = { deferSchemaDdl: true, deferredDdlUnsupported: 'skip-schema-sync' }; + +describe('both ceremony boots declare the skip-schema-sync posture', () => { + it('the plan boot: deferred, read-only, and skip-schema-sync where deferral is refused', async () => { + await runJson(['--json', '--out', join(PLAN_DIR, 'never-written.json')]); + expect(boot).toHaveBeenCalledTimes(1); + expect(boot.mock.calls[0]![0]).toEqual(expect.objectContaining({ ...POSTURE, readOnlyProbe: true })); + }, CASE_TIMEOUT_MS); + + it('the apply boot: deferred, the served composition, and skip-schema-sync where deferral is refused', async () => { + await runJson(['--apply', '--plan-file', PLAN_FILE, '--backup-taken', '--json', '--out', join(PLAN_DIR, 'never-report.json')]); + expect(boot).toHaveBeenCalledTimes(1); + expect(boot.mock.calls[0]![0]).toEqual(expect.objectContaining({ ...POSTURE, composeHostStack: true })); + }, CASE_TIMEOUT_MS); +}); + +describe('the apply reads the ledger from the catalog when its boot skipped schema sync', () => { + /** A booted stack as `bootSchemaStack` answers it, over a raw-SQL seam that answers `catalogRows`. */ + function stackWith(schemaSync: 'skipped' | 'deferred', opts: { catalogRows: unknown[]; tableAbsent: boolean }) { + const execute = vi.fn(async (sql: string) => (/pragma_table_info/.test(sql) ? opts.catalogRows : [])); + return { + execute, + stack: { + schemaSync, + driver: { name: 'com.objectstack.driver.turso', dialectName: 'sqlite', config: { client: 'better-sqlite3' }, execute }, + dbLabel: 'libsql://tenant.example.turso.io', + tableAbsent: vi.fn(() => opts.tableAbsent), + kernel: { getService: () => { throw new Error('no kernel in this pin'); } }, + composition: { securityPlugin: false }, + allObjects: () => [], + shutdown: async () => {}, + }, + }; + } + const apply = () => runJson(['--apply', '--plan-file', PLAN_FILE, '--backup-taken', '--json', '--out', join(PLAN_DIR, 'never-report.json')]); + + it('skipped, and the catalog has no sys_migration: refused ledger-absent, whatever tableAbsent says', async () => { + const { stack, execute } = stackWith('skipped', { catalogRows: [], tableAbsent: false }); + boot.mockImplementation(async () => stack); + const { payload } = await apply(); + expect(payload).toMatchObject({ error: 'apply_refused', reason: 'ledger-absent', table: 'sys_migration' }); + expect(execute).toHaveBeenCalledWith(expect.stringContaining('pragma_table_info'), ['sys_migration']); + expect(stack.tableAbsent).not.toHaveBeenCalled(); + }, CASE_TIMEOUT_MS); + + it('skipped, and the catalog has sys_migration: not refused for the ledger', async () => { + const { stack } = stackWith('skipped', { catalogRows: [{ column_name: 'id' }], tableAbsent: true }); + boot.mockImplementation(async () => stack); + const { payload } = await apply(); + expect(payload.reason).not.toBe('ledger-absent'); + }, CASE_TIMEOUT_MS); + + it('the control: a deferred boot answers from tableAbsent, without a catalog read', async () => { + const { stack, execute } = stackWith('deferred', { catalogRows: [{ column_name: 'id' }], tableAbsent: true }); + boot.mockImplementation(async () => stack); + const { payload } = await apply(); + expect(payload).toMatchObject({ error: 'apply_refused', reason: 'ledger-absent' }); + expect(execute).not.toHaveBeenCalledWith(expect.stringContaining('pragma_table_info'), ['sys_migration']); + }, CASE_TIMEOUT_MS); +}); diff --git a/packages/cli/src/utils/schema-migrate.remote-posture.test.ts b/packages/cli/src/utils/schema-migrate.remote-posture.test.ts new file mode 100644 index 00000000000..499fd344a8e --- /dev/null +++ b/packages/cli/src/utils/schema-migrate.remote-posture.test.ts @@ -0,0 +1,133 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #23816 — the two pieces of `bootSchemaStack`'s `deferredDdlUnsupported: + * 'skip-schema-sync'` posture that decide it, pinned without a database: + * + * - `transportRefusesDeferredDdl` answers `true` exactly for the transport + * whose driver refuses deferred DDL (a `turso` datasource whose URL + * `TursoDriver.detectMode` reads as `remote`), asked of the driver package, + * and `false` for every other resolved database; + * - `DeferSchemaDdlPlugin({ exemptPrimary: true })` leaves the engine's + * DEFAULT driver un-armed and arms every other driver — the ones it finds + * at `init()` and the ones registered afterwards — exactly as without the + * exemption; without it, a primary that refuses the arm fails the boot. + * + * The end-to-end case (a real `sqld`, the real command) is + * `organization-ownership.remote-face.integration.test.ts`, which runs only + * where `OS_TEST_SQLD_BIN` names a server binary; these hold the CLI half in + * every CI run. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { DeferSchemaDdlPlugin, transportRefusesDeferredDdl, type SqlDriverLike } from './schema-migrate.js'; + +describe('transportRefusesDeferredDdl', () => { + it.each([ + ['turso', 'libsql://tenant.example.turso.io', true], + ['turso', 'libsql://127.0.0.1:8080?tls=0', true], + ['turso', 'https://tenant.example.turso.io', true], + ['turso', 'file:/tmp/os-remote-posture.db', false], + ['turso', ':memory:', false], + ['sqlite', 'file:/tmp/os-remote-posture.db', false], + ['postgres', 'postgres://localhost/app', false], + ['mysql', 'mysql://localhost/app', false], + ] as const)('%s %s -> %s', async (driver, url, expected) => { + expect(await transportRefusesDeferredDdl({ driver, url })).toBe(expected); + }); +}); + +/** A driver whose arm the remote face refuses, as `TursoDriver.setDeferredDdl` does there. */ +function refusingDriver(name: string) { + return { + name, + setDeferredDdl: vi.fn((deferred: boolean) => { + if (deferred) throw Object.assign(new Error('Deferred schema DDL is not supported by the Turso REMOTE transport'), { code: 'NOT_IMPLEMENTED', status: 501 }); + }), + detectManagedDrift: vi.fn(async () => []), + applyMigrationEntries: vi.fn(async () => ({ applied: [], skipped: [] })), + }; +} +/** A driver that defers. */ +function deferringDriver(name: string) { + return { + name, + setDeferredDdl: vi.fn(), + detectManagedDrift: vi.fn(async () => []), + applyMigrationEntries: vi.fn(async () => ({ applied: [], skipped: [] })), + }; +} + +/** A plugin ctx over `services`, with an engine whose default driver is `primary`. */ +function ctxWith(primary: unknown, others: Record) { + const byName: Record = { primary, ...others }; + const engine = { + registerDriver: vi.fn(), + getDefaultDriverName: () => 'primary', + getDriverByName: (n: string) => byName[n], + getDriverForObject: () => primary, + registry: { getAllObjects: () => [{ name: 'sys_job' }] }, + }; + const services = new Map([ + ['objectql', engine], + ['driver.com.objectstack.driver.turso', primary], + ...Object.entries(others).map(([n, d]) => [`driver.${n}`, d] as [string, unknown]), + ]); + return { + engine, + ctx: { + getService: (name: string) => { + if (!services.has(name)) throw new Error(`no service ${name}`); + return services.get(name); + }, + getServices: () => services, + logger: { debug: () => {} }, + }, + }; +} + +describe('DeferSchemaDdlPlugin — the primary exemption', () => { + it('exemptPrimary: the primary is left un-armed, every other driver is armed, at init and after', async () => { + const primary = refusingDriver('remote'); + const other = deferringDriver('telemetry'); + const later = deferringDriver('declared'); + const { ctx, engine } = ctxWith(primary, { other }); + const plugin = new DeferSchemaDdlPlugin({ exemptPrimary: true }); + + await plugin.init(ctx); + + expect(primary.setDeferredDdl).not.toHaveBeenCalled(); + expect(other.setDeferredDdl).toHaveBeenCalledWith(true); + expect(plugin.drivers).toEqual([other]); + + // A driver handed to the engine afterwards is armed before the engine holds + // it; the primary handed again is still not. + (engine as unknown as { registerDriver: (d: unknown) => unknown }).registerDriver(later); + (engine as unknown as { registerDriver: (d: unknown) => unknown }).registerDriver(primary); + expect(later.setDeferredDdl).toHaveBeenCalledWith(true); + expect(primary.setDeferredDdl).not.toHaveBeenCalled(); + expect(plugin.drivers).toEqual([other, later] as unknown as SqlDriverLike[]); + plugin.release(); + }); + + it('the control: without the exemption, a primary that refuses the arm fails the boot', async () => { + const primary = refusingDriver('remote'); + const { ctx } = ctxWith(primary, {}); + const plugin = new DeferSchemaDdlPlugin(); + + await expect(plugin.init(ctx)).rejects.toThrow('Deferred schema DDL is not supported by the Turso REMOTE transport'); + expect(primary.setDeferredDdl).toHaveBeenCalledWith(true); + plugin.release(); + }); + + it('exemptPrimary exempts only the engine\'s default driver: another refusing driver still fails the boot', async () => { + const primary = deferringDriver('local'); + const remoteSibling = refusingDriver('remote'); + const { ctx } = ctxWith(primary, { remoteSibling }); + const plugin = new DeferSchemaDdlPlugin({ exemptPrimary: true }); + + await expect(plugin.init(ctx)).rejects.toThrow('Deferred schema DDL is not supported by the Turso REMOTE transport'); + expect(primary.setDeferredDdl).not.toHaveBeenCalled(); + plugin.release(); + }); +}); diff --git a/packages/cli/src/utils/schema-migrate.ts b/packages/cli/src/utils/schema-migrate.ts index 3c251f3b490..2eff2804a7c 100644 --- a/packages/cli/src/utils/schema-migrate.ts +++ b/packages/cli/src/utils/schema-migrate.ts @@ -384,8 +384,11 @@ async function planPackageTable( * {@link drivers} keeps what was armed, so the stack can report every * datasource's held-back work and flush all of it on the operator's say-so. * No driver changes: each one keeps its own deferred set and its own flush. + * + * Exported for its unit pin (`schema-migrate.remote-posture.test.ts`, #23816), + * which holds the primary exemption without booting a kernel. */ -class DeferSchemaDdlPlugin { +export class DeferSchemaDdlPlugin { name = 'com.objectstack.cli.defer-schema-ddl'; version = '1.0.0'; /** Ordering, not optionality: our init must follow the one that registers `driver.*`. */ @@ -530,8 +533,9 @@ function defaultDriverOf(ctx: { getService?: (name: string) => unknown }): unkno * Loaded lazily, as every driver is from this module (#5726). A package that * cannot be loaded answers `false`, and the boot then fails where it always * has, in `@objectstack/runtime`'s loader, naming the package to install. + * Exported for its unit pin (`schema-migrate.remote-posture.test.ts`). */ -async function transportRefusesDeferredDdl(resolved: { driver: string; url: string }): Promise { +export async function transportRefusesDeferredDdl(resolved: { driver: string; url: string }): Promise { if (resolved.driver !== 'turso') return false; let detectMode: ((config: { url: string }) => string) | undefined; try { From cc73b8090ab699b8660bd5fd21cd43662446afc6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 08:46:06 +0000 Subject: [PATCH 09/10] chore(changeset): the OS_ARTIFACT_URL gate keeps warn-and-boot on a Turso remote primary The maintainer's ruling on the contract review: say plainly that the boot migration gate does not judge a remote database. It warns and boots, applies nothing and refuses nothing, and decides that from the driver's declared driftReportCaveat. It stays that way until the remote sync honours storage.notNull and os migrate apply has a remedy on that transport. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01Lr3rg6qTp9r38exA5DHEEH --- .changeset/23816-turso-remote-ownership-ceremony.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.changeset/23816-turso-remote-ownership-ceremony.md b/.changeset/23816-turso-remote-ownership-ceremony.md index b239725cda6..bc291c3d4cd 100644 --- a/.changeset/23816-turso-remote-ownership-ceremony.md +++ b/.changeset/23816-turso-remote-ownership-ceremony.md @@ -13,8 +13,11 @@ Clause-②: yes (widening) - **`@objectstack/driver-turso`**, remote face: - `detectManagedDrift()` answers instead of refusing with `NOT_IMPLEMENTED` / 501. The verdict is the shared differ's, the one the local face runs; the physical schema it diffs is read from the remote catalog (`pragma_table_info`, `sqlite_master`, `pragma_index_list` / `pragma_index_info`, and the ADR-0120 D4 duplicate probe). The declarations it diffs against are recorded by every remote schema door and by the DDL-free `registerObjectMetadata`, so a boot that skipped schema sync is covered too. - `applyMigrationEntries()` applies `drop_index`, `create_index`, `recreate_index`, `replace_unique_index` and `drop_column` instead of refusing. Each entry runs as one libSQL `write` batch, so it lands whole or not at all: a `drop_column` drops the column's indexes with it in the same transaction, and a re-created index that fails to build leaves the old one in place. An entry is reported `applied` only once a read-back shows its target state. Every other column entry (nullability, width, a dropped DEFAULT) needs the SQLite table rebuild, which this transport does not run, so it is reported `skipped` and logged, as the in-place dialects report an op they cannot perform. + - The remote face declares `driftReportCaveat`: a sentence saying its drift report is for reading only and naming the reason. The OS_ARTIFACT_URL boot gate reads it (below). The local and embedded-replica faces declare `null`. - The remote driver's `config` now carries its `url`, so the CLI labels a remote database by its URL (credential-free) instead of `better-sqlite3`. The ceremony writes that label into its plan and refuses an apply whose plan names another database; that refusal can now tell two remote databases apart. - **`@objectstack/cli`**: the ceremony's plan and apply boots skip schema sync, instead of arming deferred DDL, when the primary datasource is a Turso remote one. No DDL is attempted, so the plan stays read-only, and every other datasource the boot connects is still deferred. The environment's own boot has already run the additive sync the ceremony reads. Every other `os migrate` command keeps refusing on that transport: their dry runs list held-back schema work, which a boot that holds nothing back cannot list. - **`@objectstack/runtime`**: `createStandaloneStack` takes `skipSchemaSync`, passed to `ObjectQLPlugin` as the option `OS_SKIP_SCHEMA_SYNC=1` names. Unset, the plugin reads the variable as before. -One behaviour change outside the ceremony: on an `os serve` boot pinned with `OS_ARTIFACT_URL`, the boot's migration gate now judges a Turso remote database as it judges every other one. It applies safe drift and refuses the boot on destructive drift. Until now it printed that it could not check the schema and booted. +On a Turso remote primary, the `OS_ARTIFACT_URL` boot migration gate on `os serve` keeps its earlier answer: cannot judge, warn and boot. It applies no drift and refuses no boot. Only its warning's wording changes, from `Could not check the physical schema` to `Did not check the physical schema against the artifact: …`, followed by the driver's reason. The gate now decides this from a capability the driver declares (`driftReportCaveat`, a string on the remote face and `null` on the local and embedded-replica faces), not from a refusal it catches. Two follow-ups have to land before that gate may act on a remote report: +- the remote transport's schema sync honours `storage.notNull`, so the report stops naming a tightening its own sync produced; +- `os migrate apply` gains a remedy on that transport for the destructive drift the gate would refuse on. From 26b6acefac48d6afbc4706c33a7835b285b1bed1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 09:01:28 +0000 Subject: [PATCH 10/10] chore(changeset): name the two remote-gate follow-ups, #23847 and #23848 Both are cited as follow-ups that this change does not close. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01Lr3rg6qTp9r38exA5DHEEH --- .changeset/23816-turso-remote-ownership-ceremony.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/23816-turso-remote-ownership-ceremony.md b/.changeset/23816-turso-remote-ownership-ceremony.md index bc291c3d4cd..e01df417435 100644 --- a/.changeset/23816-turso-remote-ownership-ceremony.md +++ b/.changeset/23816-turso-remote-ownership-ceremony.md @@ -18,6 +18,6 @@ Clause-②: yes (widening) - **`@objectstack/cli`**: the ceremony's plan and apply boots skip schema sync, instead of arming deferred DDL, when the primary datasource is a Turso remote one. No DDL is attempted, so the plan stays read-only, and every other datasource the boot connects is still deferred. The environment's own boot has already run the additive sync the ceremony reads. Every other `os migrate` command keeps refusing on that transport: their dry runs list held-back schema work, which a boot that holds nothing back cannot list. - **`@objectstack/runtime`**: `createStandaloneStack` takes `skipSchemaSync`, passed to `ObjectQLPlugin` as the option `OS_SKIP_SCHEMA_SYNC=1` names. Unset, the plugin reads the variable as before. -On a Turso remote primary, the `OS_ARTIFACT_URL` boot migration gate on `os serve` keeps its earlier answer: cannot judge, warn and boot. It applies no drift and refuses no boot. Only its warning's wording changes, from `Could not check the physical schema` to `Did not check the physical schema against the artifact: …`, followed by the driver's reason. The gate now decides this from a capability the driver declares (`driftReportCaveat`, a string on the remote face and `null` on the local and embedded-replica faces), not from a refusal it catches. Two follow-ups have to land before that gate may act on a remote report: -- the remote transport's schema sync honours `storage.notNull`, so the report stops naming a tightening its own sync produced; -- `os migrate apply` gains a remedy on that transport for the destructive drift the gate would refuse on. +On a Turso remote primary, the `OS_ARTIFACT_URL` boot migration gate on `os serve` keeps its earlier answer: cannot judge, warn and boot. It applies no drift and refuses no boot. Only its warning's wording changes, from `Could not check the physical schema` to `Did not check the physical schema against the artifact: …`, followed by the driver's reason. The gate now decides this from a capability the driver declares (`driftReportCaveat`, a string on the remote face and `null` on the local and embedded-replica faces), not from a refusal it catches. Two follow-ups, filed separately and not closed by this change, have to land before that gate may act on a remote report: +- #23847: the remote transport's schema sync honours `storage.notNull` on create and add column, so the report stops naming a tightening its own sync produced; +- #23848: `os migrate apply` works on a Turso remote database, so the destructive drift the gate would refuse on has a remedy there.