diff --git a/.changeset/v18-ceremony-dev-databases.md b/.changeset/v18-ceremony-dev-databases.md new file mode 100644 index 00000000000..8debe93fbf0 --- /dev/null +++ b/.changeset/v18-ceremony-dev-databases.md @@ -0,0 +1,18 @@ +--- +"@objectstack/cli": minor +"@objectstack/runtime": minor +--- + +The v18 organization-ownership ceremony now completes on a long-lived 17.x database, and `os dev` runs it for you on a local development database (ADR-0131, amendment 2026-10-16). + +**`os dev` upgrades a v17-shape development database in place.** When the boot refuses a local SQLite development database that is still in the v17 organization-ownership shape, `os dev` first copies the database and its `telemetry` sibling (with `-wal` / `-shm`) into `/backups/organization-ownership-/`. It then runs `os migrate organization-ownership --dev` (the plan), applies that plan, and starts the server again. The plan, the apply report and the ceremony's log are kept beside the copy. If a step blocks, `os dev` stops and prints the blocker, where the copy is, and how to resume. `os serve` / `os start`, and any database that is not a local SQLite file, keep the manual ceremony and the boot refusal unchanged. + +**What the ceremony decides for you is printed as a warning before it writes anything.** The plan carries a `warnings` list. The plan prints it, `--apply` prints it again before its first write, and the apply report keeps it. + +**The `telemetry` sibling is covered.** A development boot (or an `OS_TELEMETRY_DB` deployment) keeps `sys_audit_log`, `sys_job_run`, `sys_notification` and the other lifecycle objects in a sibling database. The ceremony now plans, attributes and drops columns for each object in the database it lives in, reading anchors across databases. Before, the boot refusal read those tables in the sibling while the ceremony only read the primary, so an upgraded database could still never boot. A copy of such a table that an older boot left in the primary is listed under `staleCopies`, warned about, and left untouched; it no longer blocks the column drops. + +**The shipped permission sets an install seeded before 2026-08-15 are adopted.** Those installs store the eight shipped sets (`organization_admin`, `member_default`, …) as `managed_by: 'admin'` rows. The authored-conversion step used to refuse them with "rename the authored row". The ceremony now re-owns them as the platform seeds they are and deletes them as mirrors once every grant names its set, so the shipped definitions apply. The warning names each row; a Setup edit to one of them is not carried over, and the pre-upgrade backup keeps it. + +**`os migrate organization-ownership --dev`** composes the stack as `os serve --dev` does: the development auth secret's fallback (so the security and audit plugins), the development `.env*` files, and the `telemetry` sibling. A development boot's refusal now names the command with `--dev`, and every refusal now names an apply the ceremony accepts: `--out PLAN` first, then `--apply --plan-file PLAN --backup-taken`. A bare `--apply` is refused without a reviewed plan file. Before, running the ceremony from a shell without `NODE_ENV=development` composed neither plugin, and its apply blocked on the shipped permission sets and on `sys_audit_log`. + +`OrganizationOwnershipBootGuardPlugin` (`@objectstack/runtime`) takes `{ dev }`; `organizationOwnershipRefusalMessage` and `assertOrganizationOwnershipMigrated` take the same option. diff --git a/content/docs/deployment/cli.mdx b/content/docs/deployment/cli.mdx index 815a00e8c27..b2abb579384 100644 --- a/content/docs/deployment/cli.mdx +++ b/content/docs/deployment/cli.mdx @@ -196,6 +196,26 @@ os dev --database file:./data/test.db --auth-secret $(openssl rand -hex 32) | `--seed-admin` / `--no-seed-admin` | — | Seed a dev admin (`admin@objectos.ai` / `admin123`) on an empty DB — default on; override with `--admin-email` / `--admin-password` | | `-v, --verbose` | — | Verbose output | +##### A 17.x development database is upgraded in place + +A v18 server refuses to start on a database still in the v17 +organization-ownership shape until the [ownership ceremony](#os-migrate-organization-ownership) +has run on it. `os dev` runs that ceremony for you when the database is a local +SQLite file: + +1. It copies the database file and its `telemetry` sibling (each with its + `-wal` / `-shm`) into `/backups/organization-ownership-/`. +2. It runs `os migrate organization-ownership --dev` (the plan), then applies + that plan. The plan, the apply report and the ceremony's log are kept + beside the copy. +3. It prints the ceremony's warnings and starts the server again. + +If a step blocks, `os dev` stops and prints the blocker, where the copy is, and +the command that resumes the run. It tries once per session. A database that is +not a local SQLite file (PostgreSQL, a remote libSQL, `:memory:`) is not +upgraded in place: run the ceremony yourself. `os serve` and `os start` never run +it; they refuse and name the command. + ##### Serving dev over https An interactive MCP client (and any OAuth client worth the name) refuses to open a @@ -1172,6 +1192,7 @@ where the data lives. | `os migrate value-shapes` | Scan stored reference and structured-JSON field values against the platform's value contract, and record the deployment's migration flag when clean | | `os migrate summary-nulls` | Backfill roll-up `count` / `sum` columns still stored as `NULL` on parent rows created before the insert-time seed. Repairs values; no flag, nothing depends on it having run | | `os migrate comment-reactions` | Carry the emoji reactions a 17.x deployment stored in the retired `sys_comment.reactions` column into `sys_comment_reaction` records, list every stored value that does not read, and record the verified run that lets `os migrate apply --allow-destructive` drop the column | +| `os migrate organization-ownership` | The v18 ownership ceremony (ADR-0131 D10): write a plan of what each table's rows will become, then `--apply` that reviewed plan. A v18 server refuses to start until it has run. See [below](#os-migrate-organization-ownership) | | `os migrate activity-changed-fields` | Stamp the two read-rule columns (`changed_fields`, `milestone_key`) on the `sys_activity` rows written before they existed. Until it runs, those rows are withheld from every reader but the system, so run it at once after upgrading. No flag; idempotent | | `os migrate meta --stored` | Replay the metadata conversion chain over this deployment's `sys_metadata` rows and rewrite the ones still carrying a pre-protocol shape. Hygiene, not a gate — nothing depends on it having run | | `os migrate security-catalog-overlays` | List the environment-wide permission-set and position rows (the legacy plurals `permissions` / `positions` included) stored over a name a configured package holds — the rows a v18 cold boot refuses — each with the package that holds its name; `--apply` deletes them, one history tombstone per row. Run it before the first v18 boot, with the flags and environment the deployment boots with: the platform security plugin's sets are held only where `os serve` composes it (an auth secret set, and the `auth` tier on), and the step takes `serve`'s `--preset` and `--dev` with `serve`'s meaning. Adopts nothing; exits 1 while it lists a row | @@ -1442,6 +1463,42 @@ the upgraded deployment's schema sync, which an `--apply` run performs itself: a dry run against a database no upgraded process has opened cannot read them, reports the failed read and exits `1`. A failed read or write exits `1`. +#### `os migrate organization-ownership` + +The v18 ownership ceremony (ADR-0131 D10). A v18 `os serve` / `os start` refuses +to start on a database still in the v17 organization-ownership shape until this +has run. Under `os dev` it runs on its own for a local SQLite database (see +[`os dev`](#os-dev)). + +```bash +os migrate organization-ownership --out plan.json # the plan: reads only +os migrate organization-ownership --apply --plan-file plan.json --backup-taken +os migrate organization-ownership --dev … # boot as `os serve --dev` does +``` + +- **The plan writes nothing.** Per table it lists the fate (column dropped, + mirror rows deleted, owner attributed, rows reported), the row counts, and the + rows whose owner cannot be derived. It refuses to plan a table it cannot read. +- **`--apply` runs only the plan you reviewed**, and only after you confirm a + backup (`--backup-taken`, or the prompt). The platform takes no backup. The run + is resumable: re-running it continues from its recorded checkpoint. +- **Warnings come before the first write.** The plan prints what the ceremony + decides on your behalf, and `--apply` prints the same warnings again before + it writes. An example: the shipped permission sets an install created before + 2026-08-15 stored as `managed_by: 'admin'` are adopted as platform seeds and + deleted, and the shipped definitions apply. A Setup edit to one of those + sets is not carried over. +- **Every database the deployment uses.** A development boot, or one with + `OS_TELEMETRY_DB`, keeps the audit, job-run and notification objects in a + `telemetry` sibling database. The ceremony plans and applies each object in + the database it lives in. If an older boot left a copy of such a table in the + primary, the ceremony lists it under `staleCopies`, warns about it, and leaves + it alone. +- **Boot it the way the deployment boots.** `--dev` composes the stack as + `os serve --dev` does: the development auth secret's fallback (and with it + the security and audit plugins), the development `.env*` files, and the + `telemetry` sibling. Pass it for a database that `os dev` serves. + #### A database created by this version needs no migration A deployment whose database the platform **creates from empty** records these diff --git a/content/docs/permissions/permission-sets.mdx b/content/docs/permissions/permission-sets.mdx index 4f9f8ab23e7..7d7fecc8ba0 100644 --- a/content/docs/permissions/permission-sets.mdx +++ b/content/docs/permissions/permission-sets.mdx @@ -398,7 +398,10 @@ posture, the platform operator. This has three author-visible consequences: An older database keeps the `sys_permission_set` table and its rows, and nothing serves or reads them. `os migrate organization-ownership --apply` (ADR-0131 D10) converts the sets an organization authored in Setup into -environment metadata under `single` and deletes the seeded copies. +environment metadata under `single` and deletes the seeded copies. The shipped +sets that an install created before 2026-08-15 stored as `managed_by: 'admin'` +count as seeded copies too. The ceremony warns about each one before it writes +anything, and then the shipped definition applies. A cross-package job function needs no hand-authored cross-package set: bind each package's own sets to one **position** (the union model adds). Where a diff --git a/docs/adr/0131-total-organization-ownership-no-null-organization-id.md b/docs/adr/0131-total-organization-ownership-no-null-organization-id.md index aa669f8a873..696e47ee1b7 100644 --- a/docs/adr/0131-total-organization-ownership-no-null-organization-id.md +++ b/docs/adr/0131-total-organization-ownership-no-null-organization-id.md @@ -1,6 +1,6 @@ # ADR-0131: Organization ownership is total — no NULL `organization_id`; declared metadata stays in code; a row exists only when an organization authored it -**Status**: Accepted (2026-09-04) — accepted by the merge that landed it on `main` ([#14976](https://github.com/objectstack-ai/objectstack/pull/14976), commit `0ed271574`), which is itself the acceptance act for a governed surface (Prime Directive #14). Execution is paused under #15193 and the maintainer's standing instruction; that pause is carried there, not by this status line. · **Amended** (2026-09-17, [#18413](https://github.com/objectstack-ai/objectstack/issues/18413) — maintainer ruling option **B** on [#18336](https://github.com/objectstack-ai/objectstack/issues/18336), 2026-09-16: **D5's assignment of `reportLegacyPlatformAdminGrant` and `resetLegacyPlatformAdminGrantReport` to D13 is withdrawn**. Those two symbols retire on the **17.x** line under [#11663](https://github.com/objectstack-ai/objectstack/issues/11663)'s design, **walled posture only**; the `single` posture keeps its zero-config first-user promotion and its grant row. **D5's Default-Organization ownership of that row is unchanged** and still lands on the v18 line under C3 — nothing about NULL ownership moves here. D14 gains one named exception. See **"Amendment (2026-09-17, #18413): the legacy-anchor retirement leaves D13 for the walled posture"** at the end.) +**Status**: Accepted (2026-09-04) — accepted by the merge that landed it on `main` ([#14976](https://github.com/objectstack-ai/objectstack/pull/14976), commit `0ed271574`), which is itself the acceptance act for a governed surface (Prime Directive #14). Execution is paused under #15193 and the maintainer's standing instruction; that pause is carried there, not by this status line. · **Amended** (2026-09-17, [#18413](https://github.com/objectstack-ai/objectstack/issues/18413) — maintainer ruling option **B** on [#18336](https://github.com/objectstack-ai/objectstack/issues/18336), 2026-09-16: **D5's assignment of `reportLegacyPlatformAdminGrant` and `resetLegacyPlatformAdminGrantReport` to D13 is withdrawn**. Those two symbols retire on the **17.x** line under [#11663](https://github.com/objectstack-ai/objectstack/issues/11663)'s design, **walled posture only**; the `single` posture keeps its zero-config first-user promotion and its grant row. **D5's Default-Organization ownership of that row is unchanged** and still lands on the v18 line under C3 — nothing about NULL ownership moves here. D14 gains one named exception. See **"Amendment (2026-09-17, #18413): the legacy-anchor retirement leaves D13 for the walled posture"** at the end.) · **Amended** (2026-10-16 — maintainer ruling on the v18 upgrade of long-lived development databases: **`os dev` runs D10's ceremony itself on a local SQLite development database**, after copying it, and boots again; `os serve` / `os start` and every other database keep the manual ceremony and the boot refusal, and **what the ceremony decides on the operator's behalf is printed as a warning before its first write**. The ceremony plans and applies **each object in the database the deployment keeps it in** (the `telemetry` sibling included), and **adopts the shipped permission sets an install seeded as `admin` rows** as the platform seeds they are. See **"Amendment (2026-10-16): the development exception, the telemetry sibling, and the legacy shipped-set seeds"** at the end.) **Deciders**: ObjectStack maintainer, 2026-09-03/04, live chat on [#13564](https://github.com/objectstack-ai/objectstack/issues/13564), verbatim and untranslated, in the order the model was built: the premise 「我理解只有代码定义的元数据是跨租户的，对象、字段、视图等 @@ -559,7 +559,8 @@ Template rows an admin has customized (`customized: true`) are the one populatio rule on (§6 Q1): keep them readable as the Default Organization's overrides (half of the D6 door), or accept the loss under the startup posture with a release note. -**The migration is a manual, operator-run ceremony — never an automatic boot step.** Maintainer, +**The migration is a manual, operator-run ceremony — never an automatic boot step** (one exception, a +local SQLite development database under `os dev` — see the 2026-10-16 amendment). Maintainer, 2026-09-04, verbatim: 「我建议 18.0 的主要考虑是客户数据变化比较大，而且需要手工执行升级脚本。」 The v18 change rewrites customer data at a scale no boot may perform silently — column drops and mirror-row deletions are irreversible, attribution rewrites ownership, the constraint locks the shape — so it ships @@ -581,6 +582,8 @@ as an `os migrate`-family command the operator runs, in this order and with thes fail-fast shape as [ADR-0093](./0093-tenancy-mode-and-membership-lifecycle.md) D5 — naming the command to run; it never auto-migrates, never runs degraded on a half-migrated schema, and offers no escape hatch that silently skips the check (an operator who wants to run 17.x semantics runs 17.x). + (`os dev` alone runs the ceremony for a refused local SQLite development database, after copying it — + the 2026-10-16 amendment; the served boot's refusal itself is unchanged.) The ceremony reuses the [ADR-0120](./0120-unique-scope-vocabulary-and-null-safe-tenant-uniqueness.md) D4 migration ceremony where it exists (index and column changes) rather than inventing a second one. @@ -1122,3 +1125,72 @@ control — the same grep over `packages/`: 15 and 17). The **assignment** is wh this amendment edits D5, D14 and §7 and leaves D13's list as it stands. The substance of the card is unaffected: before this amendment the retirement was D13's and therefore v18's; after it, it is #11663's and 17.x's. + +## Amendment (2026-10-16): the development exception, the telemetry sibling, and the legacy shipped-set seeds + +**Authority.** Maintainer ruling, taken in a Claude Code session on 2026-10-16 after the v18 ceremony +refused to complete on two long-lived 17.x development databases (`hotcrm`'s and the showcase's). The +maintainer's words, verbatim and untranslated, in the order they were given: + +> 平台的bug要修正啊，要不然客户的项目也启动不了。 +> +> 我关注的尤其是开发环境的临时数据不能让客户很麻烦。 +> +> 生产环境的数据又不能出问题，需要提示告警。 + +and the options the maintainer selected, verbatim: 「os dev 自动完成（推荐）」 for a v17-shape local +development database; 「自动当作种子副本」 for the shipped permission sets stored as `admin` rows; +「完整覆盖：每个对象在它所在的库里处理（推荐）」 for the `telemetry` sibling; 「migrate 命令加 --dev，开发启动的拒绝 +提示带上它（推荐）」 for the ceremony's composition. + +Recorded here under **Prime Directive #13**: D10 said **never an automatic boot step**, and this +amendment makes one exception to it. The record stays **Accepted**. + +### What was measured + +On both databases the ceremony stopped, in sequence, at: + +1. `sys_scim_provider` — retired before the first census, its table left in place by ruling — had no + inventory row, so the plan refused every database created before its retirement (fixed beside this + amendment as an inventory row, fate 4). +2. A ceremony run from a shell boots unlike `os dev`: no development auth secret, so no security + plugin, so the authored-conversion step could not resolve the shipped permission sets. +3. The eight shipped permission sets an install created before the 2026-08-15 seeder ruling stores as + `managed_by: 'admin'` (the object's `defaultValue`) read as Setup-authored, and the step refused each + one whose stored body differs from the shipped definition, prescribing a rename. +4. The lifecycle-classed objects a development boot keeps in the `telemetry` sibling (ADR-0057 §3.6) + were never planned there: the boot refusal reads their columns in the sibling, the ceremony read only + the primary, so a database could pass every step and still never boot; and copies of those tables a + boot from before the sibling left in the primary blocked the column drops instead. + +### What the ruling decided + +1. **The development exception.** When the guard refuses the `serve --dev` child of `os dev`, the child + says so to its parent and exits; `os dev` copies the database file and its `telemetry` sibling (each + with `-wal` / `-shm`) into `/backups/organization-ownership-/`, runs the ceremony + — the read-only plan, then `--apply` of that plan — with `--dev`, and boots again only when every step + finished. A blocked step ends `os dev` with the blocker, the copy's location and how to resume; it is + attempted once per session. A database that is not a local SQLite file is not upgraded in place. + `os serve` / `os start` keep item 5 exactly: the refusal, naming the ceremony. +2. **Warnings in production.** What the ceremony decides on the operator's behalf — the legacy seeds it + adopts, the leftover copies it leaves alone — is a `warnings` list in the plan, printed by the plan and + again by `--apply` before its first write, and kept in the apply report. +3. **The legacy shipped-set seeds are adopted.** A `sys_permission_set` row named as one of the security + plugin's shipped sets and stored as `managed_by: 'admin'` is re-owned `platform` at the start of the + authored-conversion step, so mirror deletion (D10 fate 2) takes it once every grant names its set and + the shipped definition stands. plugin-security's own resync keeps skipping such rows; the ceremony is + the one place they are re-owned, and its warning names each row and where the pre-upgrade copy is read. +4. **Each object in its own database.** The plan reads every database the deployment keeps objects in — + the primary and the `telemetry` sibling — and plans each object where the booted engine routes it, + its anchors read where theirs live; attribution writes and column drops land in that database. A + table in a database its object no longer lives in, beside the one it does, is listed (`staleCopies`) + and warned about, and no step touches it. +5. **`--dev` on the ceremony.** `os migrate organization-ownership --dev` composes as `os serve --dev` + does (development auth secret, `.env*` files, the `telemetry` sibling) — the security plugin and the + audit plugin included — and a development boot's refusal names the command with `--dev`. + +### Text changed by this amendment + +- **Status line** — the second `**Amended**` note. +- **D10** — the bold sentence and item 5 each name the exception. + diff --git a/packages/cli/src/commands/dev.ts b/packages/cli/src/commands/dev.ts index c31d6cd0822..48c183c3d30 100644 --- a/packages/cli/src/commands/dev.ts +++ b/packages/cli/src/commands/dev.ts @@ -16,6 +16,7 @@ import { assessArtifactStaleness, formatMtimeGap, } from '../utils/dev-restart.js'; +import { isOrganizationOwnershipRequiredMessage, upgradeDevelopmentDatabase } from '../utils/dev-organization-ownership-upgrade.js'; import { childEnvWithResolvedArtifact } from '../utils/internal-artifact-channel.js'; // THE artifact precedence, written once (#21501) — shared with `start`, and // with the `serve` child this command spawns. ⛔ No rung of it is restated here. @@ -742,6 +743,13 @@ export default class Dev extends Command { // load is the one path that applies the whole artifact, so the // coordinator restarts the serve child after each successful rebuild // (nodemon-style, default-on; opt out with --no-restart). + // [ADR-0131 D10, development exception] A serve child refused for the + // v17 organization-ownership shape says so before it exits; the + // coordinator then asks `recover` below, which copies the development + // database, runs the ceremony on it and boots the server again — once + // per session (`utils/dev-organization-ownership-upgrade.ts`). + let ownershipUpgrade: 'none' | 'pending' | 'attempted' = 'none'; + let upgradedBoot = false; const spawnServeChild = (info: { restartIndex: number }) => { const child = spawn( process.execPath, @@ -774,12 +782,21 @@ export default class Dev extends Command { // announcement carries no port and has nothing to do with the block // below. See {@link forwardSeedSettledToParent}. if (forwardSeedSettledToParent(msg)) return; + if (isOrganizationOwnershipRequiredMessage(msg)) { + if (ownershipUpgrade === 'none') ownershipUpgrade = 'pending'; + return; + } if (msg?.type === 'objectstack:listening' && msg.port) { const actual = String(msg.port); if (actual !== requestedPort) { console.log(chalk.dim(` ↪ server bound to port ${actual} (requested ${requestedPort})`)); } - if (info.restartIndex > 0) { + if (upgradedBoot) { + // The first child to listen after the upgrade: the boot the + // developer started, so it gets the initial boot's hint below. + upgradedBoot = false; + console.log(chalk.green(' ✓ server started on the upgraded database')); + } else if (info.restartIndex > 0) { // A restarted child is listening — the running server matches // dist/objectstack.json again. The MCP hint below is boot // noise on a restart, so it prints on the initial boot only. @@ -816,6 +833,32 @@ export default class Dev extends Command { const coordinator = new ServeRestartCoordinator({ spawnChild: spawnServeChild, exitParent: (code) => process.exit(code), + recover: async () => { + if (ownershipUpgrade !== 'pending') return false; + ownershipUpgrade = 'attempted'; + console.log(chalk.yellow( + ' ⟳ This development database is still in the v17 organization-ownership shape — upgrading it in place ' + + '(a copy is kept first)…', + )); + const { resolveStandaloneDatabase } = await import('@objectstack/runtime'); + let sqliteFile: string | null = null; + try { + sqliteFile = resolveStandaloneDatabase({ projectRoot: process.cwd(), databaseUrl: effectiveDb }).sqliteFile; + } catch { + sqliteFile = null; + } + const result = upgradeDevelopmentDatabase({ + binPath, + databaseUrl: effectiveDb, + sqliteFile, + env: localEnv, + cwd: process.cwd(), + }); + for (const warning of result.warnings) console.log(chalk.yellow(` ⚠ ${warning}`)); + for (const line of result.lines) console.log(result.ok ? chalk.green(` ✓ ${line}`) : chalk.red(` ✗ ${line}`)); + if (result.ok) upgradedBoot = true; + return result.ok; + }, log: (line) => { const trimmed = line.trimStart(); if (trimmed.startsWith('✗')) console.log(chalk.red(line)); diff --git a/packages/cli/src/commands/migrate/organization-ownership.ts b/packages/cli/src/commands/migrate/organization-ownership.ts index 5cc02a632d6..dbfc739aa93 100644 --- a/packages/cli/src/commands/migrate/organization-ownership.ts +++ b/packages/cli/src/commands/migrate/organization-ownership.ts @@ -18,7 +18,8 @@ import { errorCodeFields, isExitSignal, } from '../../utils/format.js'; -import { bootSchemaStack } from '../../utils/schema-migrate.js'; +import { bootSchemaStack, describeTelemetryDatabase } from '../../utils/schema-migrate.js'; +import { TELEMETRY_DATASOURCE } from '../../utils/telemetry-datasource.js'; import { OCCUPANCY_STOP_HINT, probeMigrationTarget } from '../../utils/migrate-occupancy-gate.js'; import { describeOccupancy } from '../../utils/sqlite-occupancy.js'; import { JOURNAL_MODE_SWITCH_NOTE } from '../../utils/sqlite-occupancy.js'; @@ -29,6 +30,7 @@ import { columnListSql, columnNameOf, ceremonyDialectOf, + PRIMARY_DATABASE, type OrganizationOwnershipPlan, } from '../../utils/organization-ownership-plan.js'; import { @@ -45,6 +47,7 @@ import { reviewedPlanFromFile, TemplateChoiceRefusal, type ApplyExecutor, + type ApplySibling, type CeremonyLedgerPort, type OrganizationOwnershipApplyReport, type ReviewedPlan, @@ -53,6 +56,57 @@ import { /** Who the ceremony's metadata saves name as their actor. */ const CEREMONY_ACTOR = 'os migrate organization-ownership'; +/** + * The databases beside the primary this boot provisioned — the `telemetry` + * sibling a development boot (`--dev`) or an `OS_TELEMETRY_DB` deployment + * keeps its lifecycle-classed objects in, exactly as the serving boot does — + * each with its raw-SQL seam, and the booted engine's routing of an object to + * its database. The ceremony plans, attributes and drops columns in each + * object's own database: the boot refusal reads every object's columns where + * the engine routes it, sibling included. + */ +async function ceremonySiblings(stack: { + kernel: { getService(name: string): unknown }; + drivers: readonly unknown[]; + telemetryDatabase: string | null; +}): Promise<{ siblings: ApplySibling[]; homeOf: (object: string) => string | undefined }> { + const { normalizeRows } = await import('@objectstack/metadata-protocol'); + const siblings: ApplySibling[] = []; + const telemetry = stack.telemetryDatabase ? stack.drivers[1] : undefined; + const exec = telemetry ? resolvePlannedDriverExec(telemetry as never) : null; + if (telemetry && exec) { + const client = (telemetry as { config?: { client?: unknown } }).config?.client; + siblings.push({ + name: TELEMETRY_DATASOURCE, + database: stack.telemetryDatabase!, + executor: { + dialect: ceremonyDialectOf(telemetry), + ...(typeof client === 'string' ? { client } : {}), + query: async (sql, params) => normalizeRows(await exec(sql, params ? [...params] : [])), + execute: async (sql, params) => { + await exec(sql, params ? [...params] : []); + }, + }, + }); + } + let engine: { getDriverForObject?: (object: string) => unknown } | undefined; + try { + engine = stack.kernel.getService('objectql') as typeof engine; + } catch { + engine = undefined; + } + const homeOf = (object: string): string | undefined => { + if (siblings.length === 0 || typeof engine?.getDriverForObject !== 'function') return undefined; + try { + const driver = engine.getDriverForObject(object) as { name?: unknown } | undefined; + return driver === telemetry || driver?.name === TELEMETRY_DATASOURCE ? TELEMETRY_DATASOURCE : PRIMARY_DATABASE; + } catch { + return undefined; + } + }; + return { siblings, homeOf }; +} + async function confirm(question: string): Promise { if (!process.stdin.isTTY) return false; // non-interactive → require --backup-taken const rl = createInterface({ input: process.stdin, output: process.stdout }); @@ -177,6 +231,7 @@ export default class MigrateOrganizationOwnership extends Command { '$ os migrate organization-ownership --database-url postgres://… --json', '$ os migrate organization-ownership --apply --plan-file plans/v18-ownership.json --backup-taken', '$ os migrate organization-ownership --fate-supplement host-fates.json', + '$ os migrate organization-ownership --dev', ]; static override flags = { @@ -191,6 +246,12 @@ export default class MigrateOrganizationOwnership extends Command { 'An existing file is never overwritten', }), json: Flags.boolean({ description: 'Also print the plan (or, with --apply, the apply report) on stdout' }), + dev: Flags.boolean({ + description: 'Compose as `os serve --dev` does: the config\'s devPlugins, the development .env files, a ' + + 'development boot for the auth secret\'s fallback, and the `telemetry` sibling database a development ' + + 'boot keeps its lifecycle objects in. Pass it when the deployment boots with --dev (`os dev`).', + default: false, + }), apply: Flags.boolean({ description: 'Perform the ceremony: attribution, the verified id-to-name rewrite, mirror deletion, column drops, then the ' + @@ -290,7 +351,7 @@ export default class MigrateOrganizationOwnership extends Command { // stop remedy alone. The plan writes no row or table and warns, as every // sibling's read-only arm does, saying what its connect can still change // on a SQLite file (`JOURNAL_MODE_SWITCH_NOTE`). - const occupancy = await probeMigrationTarget(flags['database-url']); + const occupancy = await probeMigrationTarget(flags['database-url'], { dev: flags.dev === true }); if (occupancy.status === 'busy' && flags.apply) { await refuse( { @@ -343,6 +404,7 @@ export default class MigrateOrganizationOwnership extends Command { jsonOutput: json, ...(flags['database-url'] ? { databaseUrl: flags['database-url'] } : {}), deferSchemaDdl: true, + serveFlags: { dev: flags.dev === true }, // [#23816] The plan needs no held-back work listed, so a transport that // cannot defer DDL boots with no schema sync instead (still read-only). deferredDdlUnsupported: 'skip-schema-sync', @@ -368,6 +430,7 @@ export default class MigrateOrganizationOwnership extends Command { const dialect = ceremonyDialectOf(stack.driver); const { normalizeRows } = await import('@objectstack/metadata-protocol'); const client = (stack.driver.config as { client?: unknown } | undefined)?.client; + const { siblings, homeOf } = await ceremonySiblings(stack); plan = await buildOrganizationOwnershipPlan({ reader: { dialect, @@ -376,6 +439,8 @@ export default class MigrateOrganizationOwnership extends Command { }, posture, database: stack.dbLabel, + siblings: siblings.map((d) => ({ name: d.name, database: d.database, reader: d.executor })), + homeOf, ...(inventory ? { inventory } : {}), }); } catch (error: unknown) { @@ -413,12 +478,14 @@ export default class MigrateOrganizationOwnership extends Command { } printInfo(`Database: ${chalk.white(plan.database)} · posture ${chalk.white(plan.posture)} · ` + `Default Organization ${plan.defaultOrganization ? chalk.white(plan.defaultOrganization.id) : chalk.yellow('none')}`); + for (const sibling of plan.siblings ?? []) printInfo(describeTelemetryDatabase(sibling.database)); console.log(''); for (const table of plan.tables) { if (table.physical === 'absent') continue; const unattributable = table.unattributable.length; + const where = table.database ? chalk.dim(` [${table.database}]`) : ''; console.log( - ` ${chalk.white(table.object.padEnd(34))} ${table.fate.padEnd(16)} ` + + ` ${chalk.white(table.object.padEnd(34))}${where} ${table.fate.padEnd(16)} ` + `rows ${String(table.rows?.total ?? 0).padStart(7)} null ${String(table.rows?.organizationNull ?? 0).padStart(6)} ` + `${unattributable > 0 ? chalk.yellow(`unattributable ${unattributable}`) : ''}` + `${table.notNull.willReceive ? chalk.green(' NOT NULL') : ''}`, @@ -441,6 +508,10 @@ export default class MigrateOrganizationOwnership extends Command { } } console.log(''); + // What the ceremony decides on the operator's behalf: said before the + // summary, and again by --apply before its first write. + for (const warning of plan.warnings) printWarning(warning); + if (plan.warnings.length) console.log(''); const { summary } = plan; printInfo( `${summary.present} table(s) planned, ${summary.absent} inventoried object(s) with no table here; ` + @@ -462,6 +533,7 @@ export default class MigrateOrganizationOwnership extends Command { 'database-url'?: string; out?: string; json?: boolean; + dev?: boolean; 'backup-taken'?: boolean; 'template-choice'?: string[]; 'metadata-choice'?: string[]; @@ -524,6 +596,7 @@ export default class MigrateOrganizationOwnership extends Command { composeHostStack: true, composeServedPlatform: true, composeAuthGatedSecurity: true, + serveFlags: { dev: flags.dev === true }, }); } catch (error: unknown) { await refuse({ reason: 'boot-failed', detail: String((error as Error)?.message ?? error) }, String((error as Error)?.message ?? error)); @@ -594,8 +667,12 @@ export default class MigrateOrganizationOwnership extends Command { }, }; const liveStack = stack; + const { siblings, homeOf } = await ceremonySiblings(stack); + if (!json) for (const sibling of siblings) printInfo(describeTelemetryDatabase(sibling.database)); report = await applyOrganizationOwnership({ executor, + siblings, + homeOf, // Whether this boot composed the security plugin: a shipped set or a // built-in position it alone declares is reported with that cause. securityPlugin: stack.composition.securityPlugin, @@ -701,7 +778,7 @@ export default class MigrateOrganizationOwnership extends Command { ledger, reviewedPlan, ...(inventory ? { inventory } : {}), - ...(json ? {} : { onProgress: (message: string) => printStep(message) }), + ...(json ? {} : { onProgress: (message: string) => printStep(message), onWarning: (message: string) => printWarning(message) }), }); } catch (error: unknown) { if (isExitSignal(error)) throw error; diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index bd66aa25de0..8f14bfdbb71 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -1030,6 +1030,24 @@ export function runtimeSeedSettlementChannels( }; } +/** + * Tell the `os dev` parent that the boot was refused for the v17 + * organization-ownership shape, and wait until the message is handed to the + * channel — the child exits right after, and a message still queued would be + * lost. Answers whether it was sent; a closed channel is the ordinary + * `os serve` case. + */ +async function announceOrganizationOwnershipRequired(tables: string[]): Promise { + const { ORGANIZATION_OWNERSHIP_REQUIRED_MESSAGE } = await import('../utils/dev-organization-ownership-upgrade.js'); + return new Promise((resolve) => { + try { + process.send!({ type: ORGANIZATION_OWNERSHIP_REQUIRED_MESSAGE, tables }, undefined, {}, (error) => resolve(!error)); + } catch { + resolve(false); + } + }); +} + const hostImporters = new Map(); /** @@ -3926,7 +3944,10 @@ export default class Serve extends Command { ); if (!hasOrganizationOwnershipGuard) { const { OrganizationOwnershipBootGuardPlugin } = await import('@objectstack/runtime'); - await kernel.use(new OrganizationOwnershipBootGuardPlugin()); + // A development boot's refusal names the ceremony with `--dev`, so the + // ceremony boots the way this one did (its secret, .env* files and + // `telemetry` sibling). + await kernel.use(new OrganizationOwnershipBootGuardPlugin({ dev: isDev })); trackPlugin('OrganizationOwnershipBootGuard'); } @@ -5580,6 +5601,19 @@ export default class Serve extends Command { } catch (error: any) { restoreOutput(); printDiagnostic(); + // [ADR-0131 D10, development exception] A development child of `os dev` + // refused for the v17 organization-ownership shape tells its parent, + // which copies the development database, upgrades it in place and boots + // it again (`utils/dev-organization-ownership-upgrade.ts`) — so the + // refusal is not printed as this boot's failure. Any other boot refuses + // as before, naming the ceremony. + if (isDev && typeof process.send === 'function') { + const { isOrganizationOwnershipBootRefusal } = await import('@objectstack/runtime'); + if (isOrganizationOwnershipBootRefusal(error)) { + const sent = await announceOrganizationOwnershipRequired([...error.tables]); + if (sent) this.exit(1); + } + } printError(error.message || String(error)); // A boot that died is when its warnings matter most, and the banner that // would normally carry them never printed (#4012). diff --git a/packages/cli/src/utils/dev-organization-ownership-upgrade.test.ts b/packages/cli/src/utils/dev-organization-ownership-upgrade.test.ts new file mode 100644 index 00000000000..8f2dedeac2f --- /dev/null +++ b/packages/cli/src/utils/dev-organization-ownership-upgrade.test.ts @@ -0,0 +1,112 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `os dev`'s in-place upgrade of a v17-shape development database (ADR-0131 + * D10, development exception): a copy of the database and its `telemetry` + * sibling first, then the ceremony's plan and the apply of that plan, both + * with `--dev`; the server boots again only on the ceremony's own "complete". + * The CLI runs are faked here — what they write is the ceremony's contract + * (`--out` files), and the real runs are the command's own suites. + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + isOrganizationOwnershipRequiredMessage, + ORGANIZATION_OWNERSHIP_REQUIRED_MESSAGE, + upgradeDevelopmentDatabase, + type DevelopmentUpgradeInput, +} from './dev-organization-ownership-upgrade.js'; + +let dir: string; +let primary: string; +let calls: string[][]; + +const NOW = new Date('2026-10-16T10:00:00.000Z'); +const BACKUP = (): string => join(dir, 'backups', 'organization-ownership-2026-10-16T10-00-00-000Z'); + +/** A fake CLI: answers `planCode`, and writes the files a real run writes. */ +function fakeRun(opts: { planCode?: number; report?: Record | null }): DevelopmentUpgradeInput['run'] { + return (args, logFile) => { + calls.push([...args]); + const out = args[args.indexOf('--out') + 1]!; + if (!args.includes('--apply')) { + writeFileSync(logFile, opts.planCode ? ' ✗ Refused (sys_mystery): sys_mystery carries organization_id …\n' : 'planned\n', { flag: 'a' }); + if (!opts.planCode) writeFileSync(out, '{}'); + return opts.planCode ?? 0; + } + if (opts.report) writeFileSync(out, JSON.stringify(opts.report)); + return opts.report?.complete ? 0 : 1; + }; +} + +const input = (run: DevelopmentUpgradeInput['run'], over: Partial = {}): DevelopmentUpgradeInput => ({ + binPath: '/cli/bin/run.js', + databaseUrl: `file:${primary}`, + sqliteFile: primary, + env: {}, + cwd: dir, + now: NOW, + run, + ...over, +}); + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'os-dev-upgrade-')); + primary = join(dir, 'dev.db'); + writeFileSync(primary, 'primary'); + writeFileSync(`${primary}-wal`, 'primary-wal'); + writeFileSync(join(dir, 'dev.telemetry.db'), 'telemetry'); + calls = []; +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe('os dev — the in-place upgrade of a v17-shape development database', () => { + it('⭐ copies the database and its telemetry sibling, plans, applies that plan, and boots on "complete"', () => { + const result = upgradeDevelopmentDatabase(input(fakeRun({ report: { complete: true, warnings: ['8 shipped permission set row(s) …'], steps: [] } }))); + expect(result).toMatchObject({ ok: true, backupDir: BACKUP(), warnings: ['8 shipped permission set row(s) …'] }); + expect(readdirSync(BACKUP()).sort()).toEqual(['apply.json', 'ceremony.log', 'dev.db', 'dev.db-wal', 'dev.telemetry.db', 'plan.json']); + expect(readFileSync(join(BACKUP(), 'dev.db-wal'), 'utf8')).toBe('primary-wal'); + expect(calls).toEqual([ + ['migrate', 'organization-ownership', '--dev', '--database-url', `file:${primary}`, '--out', join(BACKUP(), 'plan.json')], + ['migrate', 'organization-ownership', '--dev', '--database-url', `file:${primary}`, '--apply', '--plan-file', join(BACKUP(), 'plan.json'), '--backup-taken', '--out', join(BACKUP(), 'apply.json')], + ]); + }); + + it('a refused plan changes nothing and boots nothing: its refusal is printed, with the copy\'s location', () => { + const result = upgradeDevelopmentDatabase(input(fakeRun({ planCode: 1 }))); + expect(result.ok).toBe(false); + expect(calls).toHaveLength(1); + expect(result.lines[0]).toContain('✗ Refused (sys_mystery)'); + expect(result.lines.at(-1)).toContain('nothing was changed'); + expect(result.lines.at(-1)).toContain(BACKUP()); + }); + + it('a blocked step boots nothing: each blocker is printed, and how to resume', () => { + const result = upgradeDevelopmentDatabase(input(fakeRun({ + report: { complete: false, warnings: [], steps: [{ step: 'authored-conversion', status: 'blocked', blockers: ['The authored permission x differs …'] }] }, + }))); + expect(result.ok).toBe(false); + expect(result.lines[0]).toBe('authored-conversion: The authored permission x differs …'); + expect(result.lines.at(-1)).toContain(`--apply --plan-file ${join(BACKUP(), 'plan.json')} --backup-taken`); + }); + + it('a database that is not a local SQLite file is not upgraded in place: the ceremony is named instead', () => { + const result = upgradeDevelopmentDatabase(input(fakeRun({}), { sqliteFile: null, databaseUrl: 'postgres://db/app' })); + expect(result.ok).toBe(false); + expect(calls).toEqual([]); + expect(existsSync(join(dir, 'backups'))).toBe(false); + expect(result.lines[0]).toContain('os migrate organization-ownership --dev'); + }); + + it('the IPC message is recognised by its type alone', () => { + expect(isOrganizationOwnershipRequiredMessage({ type: ORGANIZATION_OWNERSHIP_REQUIRED_MESSAGE, tables: ['sys_job'] })).toBe(true); + expect(isOrganizationOwnershipRequiredMessage({ type: 'objectstack:listening', port: 3000 })).toBe(false); + expect(isOrganizationOwnershipRequiredMessage(null)).toBe(false); + }); +}); diff --git a/packages/cli/src/utils/dev-organization-ownership-upgrade.ts b/packages/cli/src/utils/dev-organization-ownership-upgrade.ts new file mode 100644 index 00000000000..fed14c3263a --- /dev/null +++ b/packages/cli/src/utils/dev-organization-ownership-upgrade.ts @@ -0,0 +1,183 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { spawnSync } from 'node:child_process'; +import { closeSync, copyFileSync, existsSync, mkdirSync, openSync, readFileSync } from 'node:fs'; +import { basename, dirname, join } from 'node:path'; +import { resolveTelemetryDbPath } from './telemetry-datasource.js'; + +/** + * `os dev`'s in-place upgrade of a development database still in the v17 + * organization-ownership shape — the ADR-0131 D10 development exception + * (ADR-0131, amendment 2026-10-16). + * + * ## Why `os dev` runs the ceremony, and `os serve` / `os start` never do + * + * The ceremony is an operator's act on a production database: its plan is + * reviewed, a backup is the operator's to take, and the boot refuses until it + * has run (`OrganizationOwnershipBootGuardPlugin`). A development database is + * a local file of working data. Making a developer stop, read a plan and + * hand-run two commands to keep working on it is the cost the ruling removed: + * when the `serve --dev` child `os dev` spawns is refused by the guard, the + * child says so on the IPC channel ({@link ORGANIZATION_OWNERSHIP_REQUIRED_MESSAGE}) + * and exits; `os dev` then runs the very same ceremony — plan, then apply with + * that plan, both with `--dev` — and boots the server again. + * + * ## What it guarantees + * + * - **A copy first.** Before the plan runs, the database file and its + * `telemetry` sibling (each with its `-wal` / `-shm`) are copied, closed — + * the refused child has exited — into + * `/backups/organization-ownership-/`, beside which the + * plan, the apply report and the ceremony's log are kept. + * - **Only a local SQLite file.** Any other database (PostgreSQL, a remote + * libSQL, `:memory:`) is not copied by a file copy, so it is not upgraded + * here: the operator runs the ceremony, as in production. + * - **The ceremony's own verdict.** The server boots again only when every + * step finished (`report.complete`). A blocked step is printed, with where + * the copy and the report are; nothing is guessed around it. + * - **Once per session.** A boot refused again after a finished upgrade ends + * `os dev`, naming the refusal; it never loops. + * - **The warnings.** What the ceremony decided on the developer's behalf + * (the plan's warnings) is printed, as the production ceremony prints it. + */ + +/** The IPC message a refused `serve --dev` child sends its `os dev` parent before exiting. */ +export const ORGANIZATION_OWNERSHIP_REQUIRED_MESSAGE = 'objectstack:organization-ownership-required' as const; + +export interface OrganizationOwnershipRequiredMessage { + type: typeof ORGANIZATION_OWNERSHIP_REQUIRED_MESSAGE; + /** The tables the guard found still carrying a physical organization column. */ + tables: string[]; +} + +export function isOrganizationOwnershipRequiredMessage(message: unknown): message is OrganizationOwnershipRequiredMessage { + return !!message && typeof message === 'object' && (message as { type?: unknown }).type === ORGANIZATION_OWNERSHIP_REQUIRED_MESSAGE; +} + +export interface DevelopmentUpgradeInput { + /** The CLI entry the ceremony runs through (`process.argv[1]` of `os dev`). */ + binPath: string; + /** The database URL the dev boot opens (`OS_DATABASE_URL` of the serve child). */ + databaseUrl: string; + /** The local SQLite file that URL opens, or `null` when it is not one. */ + sqliteFile: string | null; + /** The serve child's environment: the ceremony boots as the server did. */ + env: NodeJS.ProcessEnv; + cwd: string; + now?: Date; + /** Run one CLI invocation, its output appended to `logFile`; answers the exit code. Injected by tests. */ + run?: (args: readonly string[], logFile: string) => number; +} + +export interface DevelopmentUpgradeResult { + /** Every ceremony step finished: boot the server again. */ + ok: boolean; + /** Where the copy, the plan, the report and the log are. */ + backupDir?: string; + /** What to print, in order: the warnings, then the verdict. */ + warnings: string[]; + lines: string[]; +} + +const CEREMONY = ['migrate', 'organization-ownership', '--dev'] as const; + +function runCli(input: DevelopmentUpgradeInput): (args: readonly string[], logFile: string) => number { + return (args, logFile) => { + const fd = openSync(logFile, 'a'); + try { + const result = spawnSync(process.execPath, [input.binPath, ...args], { + cwd: input.cwd, + env: input.env, + stdio: ['ignore', fd, fd], + }); + return result.status ?? 1; + } finally { + closeSync(fd); + } + }; +} + +/** The `✗` lines the ceremony printed: its refusal, in its own words. */ +function refusalLines(logFile: string): string[] { + if (!existsSync(logFile)) return []; + return readFileSync(logFile, 'utf8') + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.startsWith('✗')) + .slice(-5); +} + +/** + * Upgrade the development database in place: copy it, plan, apply the plan. + * Synchronous on purpose — nothing else may open the files while it runs. + */ +export function upgradeDevelopmentDatabase(input: DevelopmentUpgradeInput): DevelopmentUpgradeResult { + const manual = + 'Run `os migrate organization-ownership --dev` (review its plan), then the same command with --apply ' + + '--plan-file FILE --backup-taken, and start os dev again.'; + if (!input.sqliteFile) { + return { + ok: false, + warnings: [], + lines: [`This development database is not a local SQLite file, so os dev cannot keep a copy of it and does not upgrade it in place. ${manual}`], + }; + } + const sibling = resolveTelemetryDbPath({ primaryPath: input.sqliteFile, env: input.env, dev: true }); + const stamp = (input.now ?? new Date()).toISOString().replace(/[:.]/g, '-'); + const backupDir = join(dirname(input.sqliteFile), 'backups', `organization-ownership-${stamp}`); + mkdirSync(backupDir, { recursive: true }); + for (const file of [input.sqliteFile, ...(sibling ? [sibling] : [])]) { + for (const suffix of ['', '-wal', '-shm']) { + if (existsSync(`${file}${suffix}`)) copyFileSync(`${file}${suffix}`, join(backupDir, `${basename(file)}${suffix}`)); + } + } + const run = input.run ?? runCli(input); + const log = join(backupDir, 'ceremony.log'); + const planFile = join(backupDir, 'plan.json'); + const applyFile = join(backupDir, 'apply.json'); + const kept = `The database as it was before the upgrade is copied to ${backupDir}, beside the ceremony's log.`; + + const planned = run([...CEREMONY, '--database-url', input.databaseUrl, '--out', planFile], log); + if (planned !== 0 || !existsSync(planFile)) { + return { + ok: false, + backupDir, + warnings: [], + lines: [...refusalLines(log), `The upgrade plan was refused, and nothing was changed. ${kept} ${manual}`], + }; + } + run([...CEREMONY, '--database-url', input.databaseUrl, '--apply', '--plan-file', planFile, '--backup-taken', '--out', applyFile], log); + if (!existsSync(applyFile)) { + return { + ok: false, + backupDir, + warnings: [], + lines: [...refusalLines(log), `The upgrade did not run to a report. ${kept} ${manual}`], + }; + } + const report = JSON.parse(readFileSync(applyFile, 'utf8')) as { + complete?: boolean; + warnings?: string[]; + steps?: Array<{ step: string; status: string; blockers?: string[] }>; + }; + const warnings = report.warnings ?? []; + if (report.complete) { + return { + ok: true, + backupDir, + warnings, + lines: [`Upgraded the development database to the v18 organization ownership. ${kept} The plan and the report are there too.`], + }; + } + const blocked = (report.steps ?? []).filter((s) => s.status === 'blocked'); + return { + ok: false, + backupDir, + warnings, + lines: [ + ...blocked.flatMap((s) => (s.blockers ?? []).map((b) => `${s.step}: ${b}`)), + `The upgrade stopped at a blocker (above); every finished step is checkpointed. ${kept} Resolve it, then run ` + + `\`os migrate organization-ownership --dev --apply --plan-file ${planFile} --backup-taken\` — or restore the copy.`, + ], + }; +} diff --git a/packages/cli/src/utils/dev-restart.test.ts b/packages/cli/src/utils/dev-restart.test.ts index 242b732131c..b3b44c8ba60 100644 --- a/packages/cli/src/utils/dev-restart.test.ts +++ b/packages/cli/src/utils/dev-restart.test.ts @@ -155,7 +155,7 @@ class FakeChild implements ServeChildLike { } } -function makeHarness(opts: { forceKillAfterMs?: number; failSpawnAt?: number } = {}) { +function makeHarness(opts: { forceKillAfterMs?: number; failSpawnAt?: number; recover?: () => Promise | boolean } = {}) { const children: FakeChild[] = []; const spawnInfos: Array<{ restartIndex: number }> = []; const exits: number[] = []; @@ -173,6 +173,7 @@ function makeHarness(opts: { forceKillAfterMs?: number; failSpawnAt?: number } = exitParent: (code) => { exits.push(code); }, log: (line) => { logs.push(line); }, forceKillAfterMs: opts.forceKillAfterMs, + ...(opts.recover ? { recover: opts.recover } : {}), }); return { coordinator, children, spawnInfos, exits, logs }; } @@ -331,4 +332,58 @@ describe('ServeRestartCoordinator', () => { expect(h.exits).toEqual([]); // replacement unaffected, parent alive expect(h.coordinator.getState()).toBe('running'); }); + + describe('recover — a child that exited on its own, repaired by the parent', () => { + it('⭐ recovered: a replacement boots and the parent survives', async () => { + const h = makeHarness({ recover: async () => true }); + h.coordinator.start(); + h.children[0].emitExit(1); + expect(h.coordinator.getState()).toBe('recovering'); + await sleep(0); + expect(h.children).toHaveLength(2); + expect(h.exits).toEqual([]); + expect(h.coordinator.getState()).toBe('running'); + }); + + it('not recovered (or the repair threw): the parent follows the child out, as before', async () => { + const h = makeHarness({ recover: async () => false }); + h.coordinator.start(); + h.children[0].emitExit(1); + await sleep(0); + expect(h.exits).toEqual([1]); + expect(h.children).toHaveLength(1); + + const h2 = makeHarness({ recover: async () => { throw new Error('repair failed'); } }); + h2.coordinator.start(); + h2.children[0].emitExit(1); + await sleep(0); + expect(h2.exits).toEqual([1]); + }); + + it('a rebuild landing during the repair is absorbed: exactly one replacement boots', async () => { + let finish!: (ok: boolean) => void; + const h = makeHarness({ recover: () => new Promise((r) => { finish = r; }) }); + h.coordinator.start(); + h.children[0].emitExit(1); + h.coordinator.requestRestart('src/a.view.ts'); + await sleep(0); // the repair is asked on the next tick + expect(h.children).toHaveLength(1); + finish(true); + await sleep(0); + expect(h.children).toHaveLength(2); + }); + + it('a signal during the repair ends the parent once the repair answers, and boots nothing', async () => { + let finish!: (ok: boolean) => void; + const h = makeHarness({ recover: () => new Promise((r) => { finish = r; }) }); + h.coordinator.start(); + h.children[0].emitExit(1); + h.coordinator.beginShutdown('SIGINT'); + await sleep(0); + finish(true); + await sleep(0); + expect(h.exits).toEqual([1]); + expect(h.children).toHaveLength(1); + }); + }); }); diff --git a/packages/cli/src/utils/dev-restart.ts b/packages/cli/src/utils/dev-restart.ts index 8099081706e..6783c73c581 100644 --- a/packages/cli/src/utils/dev-restart.ts +++ b/packages/cli/src/utils/dev-restart.ts @@ -170,9 +170,18 @@ export interface ServeRestartCoordinatorOptions { * the restart loop forever. */ forceKillAfterMs?: number; + /** + * A child that exited on its own may be recoverable: the parent repairs + * what made it exit (`os dev` upgrades a development database the boot + * refused) and answers `true` to boot a replacement, or `false` to follow + * the child out as before. Asked once per such exit; a rebuild arriving + * meanwhile is absorbed (the replacement boots the newest build), and a + * signal meanwhile ends the parent once the answer is in. + */ + recover?: (exitCode: number | null, signal: NodeJS.Signals | null) => Promise | boolean; } -export type ServeRestartState = 'idle' | 'running' | 'stopping' | 'shutdown'; +export type ServeRestartState = 'idle' | 'running' | 'stopping' | 'recovering' | 'shutdown'; /** * Supervises the single serve child of `objectstack dev` (#5148, nodemon @@ -217,7 +226,7 @@ export class ServeRestartCoordinator { * running server matches `dist/objectstack.json` again. */ requestRestart(reason: string): void { - if (this.state === 'shutdown' || this.state === 'idle') return; + if (this.state === 'shutdown' || this.state === 'idle' || this.state === 'recovering') return; if (this.state === 'stopping') { // Coalesce: the restart already in flight boots from disk, which // now holds the newer build. @@ -311,10 +320,37 @@ export class ServeRestartCoordinator { this.spawn(); return; } - // Exited on its own (crash, or killed externally). Keep the - // pre-#5148 contract — the parent follows the child — and say so - // when the dead child was a restarted one, so a rebuild that boots - // into a crash is loud instead of a silently vanished dev session. + // Exited on its own (crash, or killed externally). The parent may + // repair what stopped it and boot a replacement ({@link recover}); + // otherwise keep the pre-#5148 contract — the parent follows the child. + const { recover } = this.opts; + if (recover) { + this.state = 'recovering'; + void Promise.resolve() + .then(() => recover(code, signal)) + .catch(() => false) + .then((recovered) => { + if (this.state === 'shutdown') { + this.opts.exitParent(code ?? 0); + return; + } + if (recovered) { + this.spawn(); + return; + } + this.followChildOut(code, signal); + }); + return; + } + this.followChildOut(code, signal); + } + + /** + * The pre-#5148 contract for a child that exited on its own — and say so + * when the dead child was a restarted one, so a rebuild that boots into a + * crash is loud instead of a silently vanished dev session. + */ + private followChildOut(code: number | null, signal: NodeJS.Signals | null): void { this.state = 'shutdown'; if (this.spawnCount > 1) { this.log( diff --git a/packages/cli/src/utils/organization-ownership-apply.siblings.test.ts b/packages/cli/src/utils/organization-ownership-apply.siblings.test.ts new file mode 100644 index 00000000000..63c4bb43a8d --- /dev/null +++ b/packages/cli/src/utils/organization-ownership-apply.siblings.test.ts @@ -0,0 +1,212 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The ADR-0131 D10 `--apply` over a deployment that keeps objects in two + * databases — the primary and the `telemetry` sibling — on an install whose + * shipped permission sets were seeded as `admin` rows before 2026-08-15: + * + * - a sibling row takes its organization from a record in the primary, and + * the write lands in the sibling; + * - a sibling table's organization column is dropped there, while the + * primary's leftover copy of it is touched by no step at all; + * - the legacy shipped-set seed is adopted (re-owned `platform`), its grant + * named, and the row deleted as a mirror — the shipped definition stands; + * - the plan's warnings reach the operator before the first write, and the + * report keeps them. + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import Database from 'better-sqlite3'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { buildOrganizationOwnershipPlan, inventoryDigest } from './organization-ownership-plan.js'; +import { + applyOrganizationOwnership, + reviewedConflictsOf, + type ApplyExecutor, + type CeremonyCheckpoint, + type CeremonyLedgerPort, + type ColumnDropPort, + type OrganizationOwnershipApplyReport, +} from './organization-ownership-apply.js'; + +let dir: string; +let primary: Database.Database; +let telemetry: Database.Database; +/** Every write, tagged with the database it went to. */ +let writes: string[]; + +const LIFECYCLE = new Set(['sys_audit_log', 'sys_notification']); +const homeOf = (object: string): string => (LIFECYCLE.has(object) ? 'telemetry' : 'primary'); + +function executor(db: Database.Database, name: string): ApplyExecutor { + return { + dialect: 'sqlite', + client: 'better-sqlite3', + async query(sql, params = []) { + return db.prepare(sql).all(...(params as unknown[])) as Record[]; + }, + async execute(sql, params = []) { + writes.push(`${name}: ${sql}`); + db.prepare(sql).run(...(params as unknown[])); + }, + }; +} + +/** The schema-drift ceremony: each column dropped in the database its object lives in. */ +const columnDrops: ColumnDropPort = { + async drop(columns) { + return columns.map(({ table, column }) => { + const name = homeOf(table); + writes.push(`${name}: ALTER TABLE ${table} DROP COLUMN ${column}`); + (name === 'telemetry' ? telemetry : primary).exec(`ALTER TABLE "${table}" DROP COLUMN "${column}"`); + return { table, column, outcome: 'dropped' as const }; + }); + }, +}; + +function ledger(): CeremonyLedgerPort { + let stored: CeremonyCheckpoint | null = null; + return { + async read() { + return stored ? (JSON.parse(JSON.stringify(stored)) as CeremonyCheckpoint) : null; + }, + async write(checkpoint) { + stored = JSON.parse(JSON.stringify(checkpoint)) as CeremonyCheckpoint; + }, + }; +} + +/** The shipped definition the catalog serves for `member_default` (what the security plugin declares). */ +const SHIPPED_MEMBER_DEFAULT = { name: 'member_default', label: 'Member', objects: { crm_deal: { allowRead: true } }, systemPermissions: [] }; + +async function apply(warnings: string[] = []): Promise { + const siblings = [{ name: 'telemetry', database: join(dir, 'dev.telemetry.db'), executor: executor(telemetry, 'telemetry') }]; + const reviewed = await buildOrganizationOwnershipPlan({ + reader: executor(primary, 'primary'), + posture: 'single', + database: join(dir, 'dev.db'), + siblings: siblings.map((d) => ({ name: d.name, database: d.database, reader: d.executor })), + homeOf, + }); + return applyOrganizationOwnership({ + executor: executor(primary, 'primary'), + siblings, + homeOf, + posture: 'single', + database: join(dir, 'dev.db'), + reviewedPlan: { + file: 'plan.json', + generatedAt: reviewed.generatedAt, + database: join(dir, 'dev.db'), + inventoryDigest: inventoryDigest(), + posture: 'single', + conflicts: reviewedConflictsOf(reviewed), + }, + catalog: { + async resolve(type, name) { + return type === 'permission' && name === 'member_default' ? { definition: SHIPPED_MEMBER_DEFAULT } : undefined; + }, + }, + door: { + async save(type, name) { + throw new Error(`no environment save is expected here (${type} ${name})`); + }, + }, + templates: { async find() { return undefined; } }, + views: { async list() { return []; } }, + lock: { async verdict() { return { editable: true }; } }, + columnDrops, + ledger: ledger(), + onWarning: (message) => warnings.push(message), + now: () => new Date('2026-10-16T00:00:00Z'), + }); +} + +const rows = (db: Database.Database, sql: string): Record[] => db.prepare(sql).all() as Record[]; +const columns = (db: Database.Database, table: string): string[] => + (db.prepare(`SELECT name FROM pragma_table_info(?)`).all(table) as { name: string }[]).map((c) => c.name); + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'os-d10-apply-siblings-')); + primary = new Database(join(dir, 'dev.db')); + telemetry = new Database(join(dir, 'dev.telemetry.db')); + writes = []; + primary.exec(` + CREATE TABLE sys_organization (id TEXT PRIMARY KEY, slug TEXT); + INSERT INTO sys_organization VALUES ('org_default', 'default'); + CREATE TABLE crm_deal (id TEXT PRIMARY KEY, organization_id TEXT, name TEXT); + INSERT INTO crm_deal VALUES ('deal_1', 'org_default', 'Acme'); + + -- member_default as a pre-2026-08-15 seeder wrote it: managed_by 'admin', + -- its body older than the shipped one — and the grant naming it by id + CREATE TABLE sys_permission_set (id TEXT PRIMARY KEY, organization_id TEXT, name TEXT, label TEXT, managed_by TEXT, + active INTEGER, object_permissions TEXT, system_permissions TEXT); + INSERT INTO sys_permission_set VALUES + ('ps_legacy', NULL, 'member_default', 'Member', 'admin', 1, '{"crm_deal":{"allowRead":true,"allowEdit":true}}', '[]'); + CREATE TABLE sys_user_permission_set (id TEXT PRIMARY KEY, organization_id TEXT, user_id TEXT, + permission_set_id TEXT, permission_set TEXT); + INSERT INTO sys_user_permission_set VALUES ('grant_1', 'org_default', 'usr_1', 'ps_legacy', NULL); + + -- left in the primary by a boot from before the sibling existed + CREATE TABLE sys_audit_log (id TEXT PRIMARY KEY, organization_id TEXT, tenant_id TEXT); + INSERT INTO sys_audit_log VALUES ('old_1', NULL, NULL); + CREATE TABLE sys_notification (id TEXT PRIMARY KEY, organization_id TEXT, source_object TEXT, source_id TEXT); + INSERT INTO sys_notification VALUES ('n_old', NULL, 'crm_deal', 'deal_1'); + `); + telemetry.exec(` + CREATE TABLE sys_audit_log (id TEXT PRIMARY KEY, organization_id TEXT, tenant_id TEXT); + INSERT INTO sys_audit_log VALUES ('al_1', 'org_default', 'org_default'); + CREATE TABLE sys_notification (id TEXT PRIMARY KEY, organization_id TEXT, source_object TEXT, source_id TEXT); + INSERT INTO sys_notification VALUES ('n_1', NULL, 'crm_deal', 'deal_1'); + `); +}); + +afterEach(() => { + primary.close(); + telemetry.close(); + rmSync(dir, { recursive: true, force: true }); +}); + +describe('ADR-0131 D10 --apply — the telemetry sibling', () => { + it('⭐ attributes and drops in the sibling, and leaves the primary\'s leftover copies untouched', async () => { + const report = await apply(); + expect(report.steps.filter((s) => s.status !== 'done').map((s) => `${s.step}: ${s.blockers.join(' | ')}`)).toEqual([]); + + // The sibling row took its organization from the record in the primary. + expect(rows(telemetry, 'SELECT id, organization_id FROM sys_notification')).toEqual([{ id: 'n_1', organization_id: 'org_default' }]); + // The sibling's audit ledger lost its column; its rows stayed. + expect(columns(telemetry, 'sys_audit_log')).toEqual(['id', 'tenant_id']); + expect(rows(telemetry, 'SELECT id FROM sys_audit_log')).toEqual([{ id: 'al_1' }]); + + // The primary's leftover copies: no step wrote to them. + expect(columns(primary, 'sys_audit_log')).toContain('organization_id'); + expect(rows(primary, 'SELECT organization_id FROM sys_notification')).toEqual([{ organization_id: null }]); + expect(writes.filter((w) => w.startsWith('primary:') && /sys_audit_log|sys_notification/.test(w))).toEqual([]); + }); +}); + +describe('ADR-0131 D10 --apply — shipped permission sets seeded as admin rows', () => { + it('⭐ adopts the seed, names its grant, and deletes it as a mirror: the shipped definition stands', async () => { + const warnings: string[] = []; + const report = await apply(warnings); + const conversion = report.steps.find((s) => s.step === 'authored-conversion')!; + expect(conversion).toMatchObject({ status: 'done', summary: expect.objectContaining({ legacySeedsAdopted: 1, converted: 0 }) }); + expect(conversion.notes.join(' ')).toContain('(member_default)'); + + expect(rows(primary, 'SELECT id FROM sys_permission_set')).toEqual([]); + expect(rows(primary, 'SELECT id, permission_set FROM sys_user_permission_set')).toEqual([{ id: 'grant_1', permission_set: 'member_default' }]); + + // Warned before the first write, and kept in the report. + expect(warnings.some((w) => w.includes("stored as managed_by 'admin' (member_default)"))).toBe(true); + expect(report.warnings).toEqual(warnings); + }); + + it('a re-run adopts nothing more: idempotent', async () => { + await apply(); + const again = await apply(); + expect(again.steps.find((s) => s.step === 'authored-conversion')?.summary.legacySeedsAdopted).toBeUndefined(); + expect(again.warnings.filter((w) => w.includes("managed_by 'admin'"))).toEqual([]); + }); +}); diff --git a/packages/cli/src/utils/organization-ownership-apply.ts b/packages/cli/src/utils/organization-ownership-apply.ts index 8d08e2fd8d7..31d05723488 100644 --- a/packages/cli/src/utils/organization-ownership-apply.ts +++ b/packages/cli/src/utils/organization-ownership-apply.ts @@ -9,6 +9,7 @@ import { import { anonymousFormIntakeCandidates, anonymousFormIntakeWithdrawnIn, hashSpec } from '@objectstack/metadata-core'; import { inventoryEntryFor, + LEGACY_SHIPPED_SET_SEEDS, type InventoryEntry, type RowPredicate, } from './organization-ownership-inventory.js'; @@ -25,6 +26,7 @@ import { TEMPLATE_OBJECT, columnNameOf, predicateSql, + PRIMARY_DATABASE, Sql, type AttributionSink, type MetadataConflict, @@ -181,6 +183,17 @@ export interface ApplyExecutor extends PlanReader { execute(sql: string, params?: readonly unknown[]): Promise; } +/** + * A database beside the primary the deployment keeps objects in — the + * `telemetry` sibling (`PlanSibling`): its tables are planned, attributed and + * have their columns dropped where they live, through its own executor. + */ +export interface ApplySibling { + name: string; + database: string; + executor: ApplyExecutor; +} + /** The security catalog read (core's `SecurityCatalogReader` satisfies it). */ export interface CatalogPort { resolve(type: 'position' | 'permission', name: string): Promise<{ definition: Readonly> } | undefined>; @@ -445,6 +458,8 @@ export interface OrganizationOwnershipApplyReport { restartedFrom?: { inventoryDigest: string; steps: CeremonyStep[] }; /** Steps an earlier run finished, read from the checkpoint. */ resumedFrom: CeremonyStep[]; + /** The plan's warnings as this run read them before its first write (`OrganizationOwnershipPlan.warnings`). */ + warnings: string[]; steps: StepReport[]; tables: TableOutcome[]; /** Every step finished. */ @@ -459,6 +474,10 @@ export interface ApplyOptions { executor: ApplyExecutor; posture: string; database: string; + /** The databases beside the primary (the `telemetry` sibling), each with its own executor. */ + siblings?: readonly ApplySibling[]; + /** The database the deployment keeps `object` in (the booted engine's routing); see the plan's `homeOf`. */ + homeOf?: (object: string) => string | undefined; catalog: CatalogPort; door: MetadataDoorPort; templates: TemplatePort; @@ -498,6 +517,8 @@ export interface ApplyOptions { now?: () => Date; /** One line per milestone, for the command's human face. */ onProgress?: (message: string) => void; + /** Each of the plan's warnings, before the first write: what the ceremony decides on the operator's behalf. */ + onWarning?: (message: string) => void; } const ORG = 'organization_id'; @@ -530,7 +551,12 @@ const text = (value: unknown): string | null => /** Shared state of one run. */ interface Run { opts: ApplyOptions; + /** The primary database's SQL seam. */ sql: Sql; + /** Each database's SQL seam and executor, by name (the primary is `PRIMARY_DATABASE`). */ + databases: Map; + /** The database each planned physical table lives in, as the latest plan read it. */ + databaseOf: Map; plan: OrganizationOwnershipPlan; columns: Map | null>; attributed: Map; @@ -584,12 +610,18 @@ async function saveDefinition(run: Run, type: 'position' | 'permission', name: s run.saved.set(`${type}|${name}`, body); } +/** The database `table` lives in: its SQL seam and executor (the primary for a table no plan placed). */ +function databaseFor(run: Run, table: string): { sql: Sql; executor: ApplyExecutor } { + return run.databases.get(run.databaseOf.get(table) ?? PRIMARY_DATABASE) ?? run.databases.get(PRIMARY_DATABASE)!; +} + async function columnsOf(run: Run, table: string): Promise | null> { if (run.columns.has(table)) return run.columns.get(table)!; const present = run.plan.tables.some((t) => t.tables.includes(table)); let cols: Set | null = null; if (present) { - const rows = await run.sql.read(table, columnListSql(run.sql.dialect), [table]); + const { sql } = databaseFor(run, table); + const rows = await sql.read(table, columnListSql(sql.dialect), [table]); cols = new Set(rows.map(columnNameOf).filter((c): c is string => c !== null)); } run.columns.set(table, cols); @@ -598,7 +630,7 @@ async function columnsOf(run: Run, table: string): Promise | null> { /** Every row of `table` matching `where`, keyset-paged by id. */ async function readAll(run: Run, table: string, select: readonly string[], where: { text: string; params: unknown[] } = { text: '1 = 1', params: [] }): Promise[]> { - const { sql } = run; + const { sql } = databaseFor(run, table); const t = sql.q(table, table); const id = sql.q('id', table); const cols = [...new Set(['id', ...select])].map((c) => `${sql.q(c, table)} AS ${sql.q(c, table)}`).join(', '); @@ -618,21 +650,30 @@ async function readAll(run: Run, table: string, select: readonly string[], where async function execute(run: Run, table: string, statement: string, params: readonly unknown[]): Promise { try { - await run.opts.executor.execute(statement, params); + await databaseFor(run, table).executor.execute(statement, params); } catch (error) { throw new Error(`The write to ${table} failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error }); } } -const planFor = (run: Run, extra: { onAttribution?: AttributionSink } = {}): Promise => - buildOrganizationOwnershipPlan({ +async function planFor(run: Run, extra: { onAttribution?: AttributionSink } = {}): Promise { + const plan = await buildOrganizationOwnershipPlan({ reader: run.opts.executor, posture: run.opts.posture, database: run.opts.database, + ...(run.opts.siblings?.length + ? { siblings: run.opts.siblings.map((d) => ({ name: d.name, database: d.database, reader: d.executor })) } + : {}), + ...(run.opts.homeOf ? { homeOf: run.opts.homeOf } : {}), ...(run.opts.inventory ? { inventory: run.opts.inventory } : {}), ...(run.opts.now ? { now: run.opts.now() } : {}), ...extra, }); + for (const table of plan.tables) { + for (const physical of table.tables) run.databaseOf.set(physical, table.database ?? PRIMARY_DATABASE); + } + return plan; +} const nullsUnderAttribution = (plan: OrganizationOwnershipPlan): number => plan.tables.filter((t) => t.fate === 'attribution').reduce((n, t) => n + (t.rows?.organizationNull ?? 0), 0); @@ -648,24 +689,28 @@ async function runAttribution(run: Run): Promise { let wrote = 0; const plan = await planFor(run, { onAttribution: async ({ object, assignments }) => { + // Each row is written in the database the plan read it from (the + // assignment says, before any plan of this pass has placed the table). const groups = new Map(); for (const a of assignments) { - const key = `${a.table}\u0000${a.organizationId}`; + const key = `${a.database ?? PRIMARY_DATABASE}\u0000${a.table}\u0000${a.organizationId}`; const ids = groups.get(key) ?? []; ids.push(a.id); groups.set(key, ids); } for (const [key, ids] of groups) { - const [table, organizationId] = key.split('\u0000') as [string, string]; - const t = run.sql.q(table, table); + const [database, table, organizationId] = key.split('\u0000') as [string, string, string]; + const { sql, executor } = run.databases.get(database)!; + const t = sql.q(table, table); for (let i = 0; i < ids.length; i += IN_CHUNK) { const chunk = ids.slice(i, i + IN_CHUNK); - await execute( - run, - table, - `UPDATE ${t} SET ${run.sql.q(ORG, table)} = ? WHERE ${run.sql.q(ORG, table)} IS NULL AND ${run.sql.q('id', table)} IN (${chunk.map(() => '?').join(', ')})`, - [organizationId, ...chunk], - ); + const statement = + `UPDATE ${t} SET ${sql.q(ORG, table)} = ? WHERE ${sql.q(ORG, table)} IS NULL AND ${sql.q('id', table)} IN (${chunk.map(() => '?').join(', ')})`; + try { + await executor.execute(statement, [organizationId, ...chunk]); + } catch (error) { + throw new Error(`The write to ${table} failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error }); + } } } wrote += assignments.length; @@ -774,6 +819,31 @@ async function runAuthoredConversion(run: Run): Promise { if (!cols || !where) return []; return readAll(run, table, [...cols], predicateSql(run.sql, table, cols, where)); }; + // The shipped sets an install seeded as `admin` rows (LEGACY_SHIPPED_SET_SEEDS) + // are the platform's seeds, not Setup-authored sets: re-owned `platform` + // here, under every posture, so the authored read below leaves them out and + // mirror deletion takes them once the rewrite has named every grant. + // Idempotent: a re-run finds none. The plan warned, naming each row. + const setColumns = await columnsOf(run, SETS); + if (setColumns?.has('managed_by') && setColumns.has('name')) { + const legacy = predicateSql(run.sql, SETS, setColumns, LEGACY_SHIPPED_SET_SEEDS); + const adopted = await readAll(run, SETS, ['name'], legacy); + if (adopted.length > 0) { + await execute( + run, + SETS, + `UPDATE ${run.sql.q(SETS, SETS)} SET ${run.sql.q('managed_by', SETS)} = ? WHERE ${legacy.text}`, + ['platform', ...legacy.params], + ); + summary.legacySeedsAdopted = adopted.length; + notes.push( + `Adopted ${adopted.length} shipped permission set row(s) stored as managed_by 'admin' ` + + `(${[...new Set(adopted.map((r) => String(r.name)))].sort().join(', ')}) as platform seeds: the shipped ` + + 'definitions stand, and mirror deletion removes these rows once every grant names its set.', + ); + run.opts.onProgress?.(`adopted ${adopted.length} legacy shipped permission set row(s) as platform seeds`); + } + } const sets = await authoredRows(SETS); const positions = await authoredRows(POSITIONS); const capabilities = await authoredRows(CAPABILITIES); @@ -2564,6 +2634,11 @@ export async function applyOrganizationOwnership(opts: ApplyOptions): Promise [d.name, { sql: new Sql(d.executor, d.executor.dialect), executor: d.executor }] as const), + ]), + databaseOf: new Map(), plan: undefined as unknown as OrganizationOwnershipPlan, columns: new Map(), attributed: new Map(), @@ -2578,6 +2653,7 @@ export async function applyOrganizationOwnership(opts: ApplyOptions): Promise 0 ? { restartedFrom: { inventoryDigest: stored.inventoryDigest, steps: superseded } } : {}), resumedFrom, + warnings: [...before.warnings], steps, tables, complete, diff --git a/packages/cli/src/utils/organization-ownership-inventory.test.ts b/packages/cli/src/utils/organization-ownership-inventory.test.ts index 3afc578072d..19bd5c16635 100644 --- a/packages/cli/src/utils/organization-ownership-inventory.test.ts +++ b/packages/cli/src/utils/organization-ownership-inventory.test.ts @@ -18,10 +18,13 @@ import { PLATFORM_OBJECTS_BY_PACKAGE, type OrganizationOwnershipSupplementEntry, } from '@objectstack/spec/system'; +import { securityDefaultPermissionSets } from '@objectstack/plugin-security'; import { HostFateSupplementRefusal, + LEGACY_SHIPPED_SET_SEEDS, ORGANIZATION_OWNERSHIP_FATES, ORGANIZATION_OWNERSHIP_INVENTORY, + SHIPPED_PERMISSION_SET_NAMES, inventoryEntryFor, mergeHostFateSupplement, readHostFateSupplement, @@ -140,6 +143,14 @@ describe('ADR-0131 D10 inventory — every censused object has one fate and a ci } }); + it('the shipped permission set names are the security plugin\'s, so every legacy admin-stamped seed is adopted', () => { + expect([...SHIPPED_PERMISSION_SET_NAMES].sort()).toEqual(securityDefaultPermissionSets.map((set) => set.name).sort()); + expect(LEGACY_SHIPPED_SET_SEEDS).toEqual([ + { column: 'managed_by', equals: 'admin' }, + { column: 'name', in: [...SHIPPED_PERMISSION_SET_NAMES] }, + ]); + }); + it('every parent and holder an anchor names is itself inventoried', () => { for (const entry of ORGANIZATION_OWNERSHIP_INVENTORY) { for (const anchor of entry.anchors ?? []) { diff --git a/packages/cli/src/utils/organization-ownership-inventory.ts b/packages/cli/src/utils/organization-ownership-inventory.ts index 25bb2ac603c..0898cad9b82 100644 --- a/packages/cli/src/utils/organization-ownership-inventory.ts +++ b/packages/cli/src/utils/organization-ownership-inventory.ts @@ -197,6 +197,35 @@ const EX: readonly InventoryCensus[] = ['example-census-2026-08-31']; const SEEDED = [{ column: 'managed_by', in: ['package', 'platform', 'system'] }] satisfies RowPredicate; const AUTHORED = [{ column: 'managed_by', notIn: ['package', 'platform', 'system'] }] satisfies RowPredicate; +/** + * The permission sets the security plugin ships (`securityDefaultPermissionSets`, + * `@objectstack/plugin-security`), by name. `organization-ownership-inventory.test.ts` + * holds this list equal to the plugin's. + */ +export const SHIPPED_PERMISSION_SET_NAMES: readonly string[] = [ + 'admin_full_access', 'organization_admin', 'organization_admin_no_bypass', 'member_default', 'viewer_readonly', + 'mcp_agent_data_read', 'mcp_agent_data_write', 'mcp_agent_restricted', +]; + +/** + * The shipped sets an install seeded before the seeder stamped its rows + * `managed_by: 'platform'` (commit 712e185db, ruled 2026-08-15): the insert + * omitted the column, so the object's `defaultValue: 'admin'` filled it, and + * every install created earlier holds the shipped sets as `admin` rows that + * read as Setup-authored. The ceremony adopts them as the platform seeds they + * are — re-owned `platform` at the authored conversion, then deleted as mirrors + * once the grants name them — so the shipped definition stands; an edit an + * administrator made to one of them in Setup is not carried, and the plan and + * the apply both warn, naming each row (ADR-0131, amendment 2026-10-16). + */ +export const LEGACY_SHIPPED_SET_SEEDS = [ + { column: 'managed_by', equals: 'admin' }, + { column: 'name', in: [...SHIPPED_PERMISSION_SET_NAMES] }, +] satisfies RowPredicate; + +/** The `sys_permission_set` category that counts {@link LEGACY_SHIPPED_SET_SEEDS}. */ +export const LEGACY_SHIPPED_SET_SEEDS_CATEGORY = 'legacy-shipped-set-seeds'; + const parent = (childKey: string, parentObject: string): Anchor => ({ kind: 'parent', childKey, parentObject }); const subject = (objectColumn: string, idColumn: string): Anchor => ({ kind: 'subject', objectColumn, idColumn }); @@ -350,7 +379,15 @@ export const ORGANIZATION_OWNERSHIP_INVENTORY: readonly InventoryEntry[] = [ fate: 'mirror-deletion', citation: `${D3_CATALOG}; ADR-0094 D1 generalized (D2); D10 fate 2`, mirror: SEEDED, - categories: [{ id: 'authored', label: 'Setup-authored permission sets — converted to environment metadata by C3, never deleted as mirrors', citation: 'ADR-0131 D3', where: AUTHORED }], + categories: [ + { id: 'authored', label: 'Setup-authored permission sets — converted to environment metadata by C3, never deleted as mirrors', citation: 'ADR-0131 D3', where: AUTHORED }, + { + id: LEGACY_SHIPPED_SET_SEEDS_CATEGORY, + label: 'shipped permission sets an install seeded before 2026-08-15 as managed_by admin — adopted as platform seeds and deleted as mirrors; a Setup edit to one of them is not carried', + citation: 'seed provenance: commit 712e185db (ruled 2026-08-15); ADR-0131, amendment 2026-10-16', + where: LEGACY_SHIPPED_SET_SEEDS, + }, + ], }, { object: 'sys_capability', diff --git a/packages/cli/src/utils/organization-ownership-plan.siblings.integration.test.ts b/packages/cli/src/utils/organization-ownership-plan.siblings.integration.test.ts new file mode 100644 index 00000000000..682120ea4db --- /dev/null +++ b/packages/cli/src/utils/organization-ownership-plan.siblings.integration.test.ts @@ -0,0 +1,187 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The ADR-0131 D10 preflight over a deployment that keeps objects in two + * databases — the primary and the `telemetry` sibling a development boot keeps + * its lifecycle-classed objects in — and over the shipped permission sets an + * install seeded as `admin` rows before 2026-08-15. + * + * - each object is planned in the database the deployment keeps it in, its + * anchors read in theirs (a notification in the sibling takes its + * organization from a record in the primary); + * - a table left in a database its object no longer lives in is listed and + * warned about, never planned: nothing reads it; + * - the legacy shipped-set seeds are counted in their own category and named + * in a warning, and a Setup-authored `admin` set is not among them; + * - the control: with no sibling (a production boot), the primary plans as + * it always did, and the run issues SELECTs only on both databases. + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import Database from 'better-sqlite3'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + buildOrganizationOwnershipPlan, + type AttributionAssignment, + type OrganizationOwnershipPlan, + type PlanReader, + type TablePlan, +} from './organization-ownership-plan.js'; + +let dir: string; +let primary: Database.Database; +let telemetry: Database.Database; +let statements: string[]; + +const LIFECYCLE = new Set(['sys_audit_log', 'sys_notification', 'sys_http_delivery']); + +function reader(db: Database.Database): PlanReader { + return { + dialect: 'sqlite', + client: 'better-sqlite3', + async query(sql, params = []) { + statements.push(sql); + return db.prepare(sql).all(...(params as unknown[])) as Record[]; + }, + }; +} + +const plan = (opts: { siblings?: boolean; onAttribution?: (batch: { object: string; assignments: AttributionAssignment[] }) => Promise } = {}): Promise => + buildOrganizationOwnershipPlan({ + reader: reader(primary), + posture: 'isolated', + database: join(dir, 'dev.db'), + ...(opts.siblings === false + ? {} + : { + siblings: [{ name: 'telemetry', database: join(dir, 'dev.telemetry.db'), reader: reader(telemetry) }], + homeOf: (object: string) => (LIFECYCLE.has(object) ? 'telemetry' : 'primary'), + }), + ...(opts.onAttribution ? { onAttribution: opts.onAttribution } : {}), + now: new Date('2026-10-16T00:00:00Z'), + }); + +const table = (p: OrganizationOwnershipPlan, object: string): TablePlan => { + const found = p.tables.find((t) => t.object === object); + if (!found) throw new Error(`no ${object} in the plan`); + return found; +}; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'os-d10-siblings-')); + primary = new Database(join(dir, 'dev.db')); + telemetry = new Database(join(dir, 'dev.telemetry.db')); + statements = []; + primary.exec(` + CREATE TABLE sys_organization (id TEXT PRIMARY KEY, slug TEXT); + INSERT INTO sys_organization VALUES ('org_default', 'default'), ('org_b', 'b'); + CREATE TABLE crm_deal (id TEXT PRIMARY KEY, organization_id TEXT, name TEXT); + INSERT INTO crm_deal VALUES ('deal_1', 'org_b', 'Acme'); + CREATE TABLE sys_webhook (id TEXT PRIMARY KEY, organization_id TEXT, managed_by TEXT); + INSERT INTO sys_webhook VALUES ('wh_1', 'org_b', 'admin'); + + -- the shipped sets as a pre-2026-08-15 seeder wrote them, a set the + -- platform seeded since, and one a Setup administrator authored + CREATE TABLE sys_permission_set (id TEXT PRIMARY KEY, organization_id TEXT, name TEXT, managed_by TEXT); + INSERT INTO sys_permission_set VALUES + ('ps_1', NULL, 'organization_admin', 'admin'), + ('ps_2', NULL, 'member_default', 'admin'), + ('ps_3', NULL, 'viewer_readonly', 'platform'), + ('ps_4', NULL, 'field_sales', 'admin'); + + -- left in the primary by a boot from before the sibling existed + CREATE TABLE sys_audit_log (id TEXT PRIMARY KEY, organization_id TEXT, tenant_id TEXT); + INSERT INTO sys_audit_log VALUES ('old_1', 'org_b', 'org_b'), ('old_2', NULL, NULL); + `); + telemetry.exec(` + -- fate 1 in the sibling: the column is still there + CREATE TABLE sys_audit_log (id TEXT PRIMARY KEY, organization_id TEXT, tenant_id TEXT); + INSERT INTO sys_audit_log VALUES ('al_1', 'org_b', 'org_b'), ('al_2', NULL, NULL); + -- fate 3 in the sibling, anchored on records in the primary + CREATE TABLE sys_notification (id TEXT PRIMARY KEY, organization_id TEXT, source_object TEXT, source_id TEXT); + INSERT INTO sys_notification VALUES ('n_1', NULL, 'crm_deal', 'deal_1'), ('n_2', 'org_b', 'crm_deal', 'deal_1'); + CREATE TABLE sys_http_delivery (id TEXT PRIMARY KEY, organization_id TEXT, ref_id TEXT); + INSERT INTO sys_http_delivery VALUES ('d_1', NULL, 'wh_1'); + `); +}); + +afterEach(() => { + primary.close(); + telemetry.close(); + rmSync(dir, { recursive: true, force: true }); +}); + +describe('ADR-0131 D10 preflight — the telemetry sibling', () => { + it('⭐ plans each lifecycle object in the sibling, its anchors read where their objects live', async () => { + const p = await plan(); + expect(p.siblings).toEqual([{ name: 'telemetry', database: join(dir, 'dev.telemetry.db') }]); + + const audit = table(p, 'sys_audit_log'); + expect(audit).toMatchObject({ database: 'telemetry', fate: 'column-drop', organizationColumn: 'present', physical: 'present' }); + expect(audit.rows).toEqual({ total: 2, organizationNull: 1, stamped: 1 }); + + const notification = table(p, 'sys_notification'); + expect(notification.database).toBe('telemetry'); + expect(notification.attribution).toEqual({ 'subject:source_object/source_id': 1 }); + expect(notification.unattributable).toEqual([]); + + const delivery = table(p, 'sys_http_delivery'); + expect(delivery.attribution).toEqual({ 'parent:ref_id->sys_webhook': 1 }); + + // A primary object carries no database tag. + expect(table(p, 'crm_deal').database).toBeUndefined(); + }); + + it('⭐ a table left in a database its object no longer lives in is warned about and never planned', async () => { + const p = await plan(); + expect(p.staleCopies).toEqual([ + { object: 'sys_audit_log', database: 'primary', home: 'telemetry', tables: ['sys_audit_log'], rows: 2 }, + ]); + expect(p.tables.filter((t) => t.object === 'sys_audit_log')).toHaveLength(1); + expect(p.warnings.some((w) => w.startsWith('sys_audit_log has a table in the primary database, but this deployment keeps the object in the telemetry database'))).toBe(true); + }); + + it('every derived owner of a sibling row names the sibling, so the apply writes it there', async () => { + const seen: AttributionAssignment[] = []; + await plan({ onAttribution: async ({ assignments }) => { seen.push(...assignments); } }); + expect(seen).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: 'n_1', table: 'sys_notification', database: 'telemetry', organizationId: 'org_b' }), + expect.objectContaining({ id: 'd_1', table: 'sys_http_delivery', database: 'telemetry', organizationId: 'org_b' }), + ])); + expect(seen.every((a) => a.database === 'telemetry')).toBe(true); + }); + + it('the control: with no sibling the primary plans as it always did, and every statement on both databases is a SELECT', async () => { + const p = await plan({ siblings: false }); + expect(p.siblings).toBeUndefined(); + expect(p.staleCopies).toEqual([]); + expect(table(p, 'sys_audit_log')).toMatchObject({ fate: 'column-drop', physical: 'present' }); + expect(table(p, 'sys_audit_log').database).toBeUndefined(); + + statements = []; + await plan(); + expect(statements.length).toBeGreaterThan(10); + expect(statements.filter((s) => !/^SELECT\b/.test(s.trim()))).toEqual([]); + }); +}); + +describe('ADR-0131 D10 preflight — shipped permission sets seeded as admin rows', () => { + it('⭐ counts them in their own category and names each in a warning; a Setup-authored set is not among them', async () => { + const p = await plan(); + const sets = table(p, 'sys_permission_set'); + expect(sets.categories.find((c) => c.id === 'legacy-shipped-set-seeds')?.rows).toBe(2); + const warning = p.warnings.find((w) => w.includes("stored as managed_by 'admin'")); + expect(warning).toContain('(member_default, organization_admin)'); + expect(warning).not.toContain('field_sales'); + expect(warning).toContain('is not carried'); + }); + + it('a database with none of them warns about nothing', async () => { + primary.exec(`UPDATE sys_permission_set SET managed_by = 'platform' WHERE name IN ('organization_admin', 'member_default')`); + const p = await plan({ siblings: false }); + expect(table(p, 'sys_permission_set').categories.find((c) => c.id === 'legacy-shipped-set-seeds')?.rows).toBe(0); + expect(p.warnings).toEqual([]); + }); +}); diff --git a/packages/cli/src/utils/organization-ownership-plan.ts b/packages/cli/src/utils/organization-ownership-plan.ts index 25cf4a26b48..ba6386b1e9c 100644 --- a/packages/cli/src/utils/organization-ownership-plan.ts +++ b/packages/cli/src/utils/organization-ownership-plan.ts @@ -6,6 +6,8 @@ import { dialectOfDriver } from '@objectstack/metadata-core'; import { hasPlatformObjectPrefix } from '@objectstack/spec/system'; import { APPLICATION_OBJECT_FATE, + LEGACY_SHIPPED_SET_SEEDS, + LEGACY_SHIPPED_SET_SEEDS_CATEGORY, ORGANIZATION_OWNERSHIP_INVENTORY, type Anchor, type InventoryCategory, @@ -103,6 +105,8 @@ export interface AttributionAssignment { id: string; /** The physical table the row lives in (a rotation shard, or the object's own table). */ table: string; + /** The database that table lives in, when it is not the primary ({@link PlanSibling.name}). */ + database?: string; organizationId: string; /** The anchor that answered, as the plan's `attribution` counts label it. */ via: string; @@ -115,6 +119,34 @@ export interface AttributionAssignment { */ export type AttributionSink = (batch: { object: string; assignments: AttributionAssignment[] }) => Promise; +/** + * A database beside the primary that the deployment keeps objects in — the + * `telemetry` sibling a development boot (or an `OS_TELEMETRY_DB` deployment) + * keeps its lifecycle-classed objects in (ADR-0057 §3.6). The plan reads its + * tables with their own fates, through their own reader. + */ +export interface PlanSibling { + /** The datasource name (`telemetry`). */ + name: string; + /** What the operator reads for it: its file. */ + database: string; + reader: PlanReader; +} + +/** A table in a database its object does not live in ({@link OrganizationOwnershipPlan.staleCopies}). */ +export interface StaleCopy { + object: string; + /** Where the table is: `primary`, or a {@link PlanSibling.name}. */ + database: string; + /** Where the object lives. */ + home: string; + tables: string[]; + rows: number; +} + +/** The name the primary database goes by beside its siblings. */ +export const PRIMARY_DATABASE = 'primary'; + export interface PlannedCategory { id: string; label: string; @@ -135,6 +167,12 @@ export interface PlannedDeparture { export interface TablePlan { object: string; + /** + * The database the object lives in, when it is not the primary: the + * `telemetry` sibling a development boot (or an `OS_TELEMETRY_DB` + * deployment) keeps its lifecycle-classed objects in ({@link PlanSibling}). + */ + database?: string; fate: OrganizationOwnershipFate; citation: string; /** `inventory` row, or the application-object default. */ @@ -398,6 +436,20 @@ export interface OrganizationOwnershipPlan { organizations: number; /** sha256 of the inventory this plan was computed from. */ inventoryDigest: string; + /** The databases beside the primary the plan read, each by name ({@link PlanSibling}). */ + siblings?: Array<{ name: string; database: string }>; + /** + * Tables in a database the object does not live in — left by a boot from + * before the object moved (a lifecycle-classed object a development boot now + * keeps in the `telemetry` sibling, say). Nothing reads them, so the ceremony + * neither attributes nor drops them; each is warned about. + */ + staleCopies: StaleCopy[]; + /** + * What the ceremony decides on the operator's behalf, one sentence each: + * printed by the plan and again by `--apply` before anything is written. + */ + warnings: string[]; tables: TablePlan[]; summary: { present: number; @@ -604,7 +656,7 @@ interface Catalog { columns: Map>; } -async function readCatalog(sql: Sql, reader: PlanReader, inventoried: ReadonlySet): Promise { +async function readCatalog(sql: Sql, reader: PlanReader, inventoried: ReadonlySet, sibling = false): Promise { const listSql = physicalTableListSql(sql.dialect); if (!listSql) { throw new OrganizationOwnershipPlanRefusal( @@ -631,7 +683,8 @@ async function readCatalog(sql: Sql, reader: PlanReader, inventoried: ReadonlySe list.push(table); bases.set(base, list); } - if (![...bases.keys()].some((base) => hasPlatformObjectPrefix(base) && inventoried.has(base))) { + // A sibling holds only the objects the deployment moved there — possibly none yet. + if (!sibling && ![...bases.keys()].some((base) => hasPlatformObjectPrefix(base) && inventoried.has(base))) { throw new OrganizationOwnershipPlanRefusal( 'not-an-objectstack-database', `This database holds none of the platform tables the inventory names (${rows.length} table(s) listed), so it is ` + @@ -660,32 +713,57 @@ async function readCatalog(sql: Sql, reader: PlanReader, inventoried: ReadonlySe return { bases, columns }; } +/** One database the plan reads: its name, its SQL seam and what its catalog answered. */ +interface PlanDatabase { + name: string; + sql: Sql; + catalog: Catalog; +} + /** Context one table's attribution reads beyond the table itself. */ interface PlanContext { + /** The database the table being planned lives in. */ + database: string; sql: Sql; catalog: Catalog; + /** Every database the plan reads, the primary first: an anchor is read where its object lives. */ + databases: readonly PlanDatabase[]; + /** The database the deployment keeps `object` in, when the boot says. */ + homeOf: (object: string) => string | undefined; posture: string; defaultOrganizationId: string | null; onAttribution?: AttributionSink; } +/** + * The database an anchor's object is read from: the one the deployment keeps + * it in when that one holds its table, else the first that does. + */ +function locate(ctx: PlanContext, object: string): PlanDatabase | null { + const home = ctx.homeOf(object); + const at = ctx.databases.find((d) => d.name === home && d.catalog.bases.has(object)); + return at ?? ctx.databases.find((d) => d.catalog.bases.has(object)) ?? null; +} + /** The organization of each id in `ids` on `object`'s own table, or a reason it cannot be read. */ async function organizationsById( ctx: PlanContext, object: string, ids: string[], ): Promise | string> { - const tables = ctx.catalog.bases.get(object); - if (!tables || tables.length !== 1 || tables[0] !== object) return `${object} has no table on this database`; - const cols = ctx.catalog.columns.get(object); + const db = locate(ctx, object); + const tables = db?.catalog.bases.get(object); + if (!db || !tables || tables.length !== 1 || tables[0] !== object) return `${object} has no table on this database`; + const cols = db.catalog.columns.get(object); if (!cols?.has(ID) || !cols.has(ORG)) return `${object} carries no ${ORG} to derive from`; const out = new Map(); - const t = ctx.sql.q(object, object); + const { sql } = db; + const t = sql.q(object, object); for (let i = 0; i < ids.length; i += IN_CHUNK) { const chunk = ids.slice(i, i + IN_CHUNK); - const rows = await ctx.sql.read( + const rows = await sql.read( object, - `SELECT ${ctx.sql.q(ID, object)} AS id, ${ctx.sql.q(ORG, object)} AS org FROM ${t} WHERE ${ctx.sql.q(ID, object)} IN (${chunk.map(() => '?').join(', ')})`, + `SELECT ${sql.q(ID, object)} AS id, ${sql.q(ORG, object)} AS org FROM ${t} WHERE ${sql.q(ID, object)} IN (${chunk.map(() => '?').join(', ')})`, chunk, ); for (const row of rows) out.set(String(row.id), nonEmpty(row.org)); @@ -712,7 +790,7 @@ async function applyAnchor( const left: PendingRow[] = []; const derive = (row: PendingRow, organizationId: string): void => { derived[label] = (derived[label] ?? 0) + 1; - assigned.push({ id: row.id, table: row.shard, organizationId, via: label }); + assigned.push({ id: row.id, table: row.shard, ...(ctx.database !== PRIMARY_DATABASE ? { database: ctx.database } : {}), organizationId, via: label }); }; if (anchor.kind === 'parent') { const keys = [...new Set(rows.map((r) => nonEmpty(r.values[anchor.childKey])).filter((k): k is string => k !== null))]; @@ -755,20 +833,22 @@ async function applyAnchor( return left; } // holder - const holderTables = ctx.catalog.bases.get(anchor.holderObject); - const cols = ctx.catalog.columns.get(anchor.holderObject); - if (!holderTables || holderTables.length !== 1 || !cols?.has(anchor.holderKey) || !cols.has(ORG)) { + const holderDb = locate(ctx, anchor.holderObject); + const holderTables = holderDb?.catalog.bases.get(anchor.holderObject); + const cols = holderDb?.catalog.columns.get(anchor.holderObject); + if (!holderDb || !holderTables || holderTables.length !== 1 || !cols?.has(anchor.holderKey) || !cols.has(ORG)) { return rows.map((row) => ({ ...row, reason: `${anchor.holderObject} holds no ${anchor.holderKey} / ${ORG} to derive from` })); } const holders = new Map>(); - const t = ctx.sql.q(anchor.holderObject, anchor.holderObject); - const k = ctx.sql.q(anchor.holderKey, anchor.holderObject); + const hsql = holderDb.sql; + const t = hsql.q(anchor.holderObject, anchor.holderObject); + const k = hsql.q(anchor.holderKey, anchor.holderObject); const ids = rows.map((r) => r.id); for (let i = 0; i < ids.length; i += IN_CHUNK) { const chunk = ids.slice(i, i + IN_CHUNK); - const found = await ctx.sql.read( + const found = await hsql.read( anchor.holderObject, - `SELECT ${k} AS k, ${ctx.sql.q(ORG, anchor.holderObject)} AS org FROM ${t} WHERE ${k} IN (${chunk.map(() => '?').join(', ')}) AND ${ctx.sql.q(ORG, anchor.holderObject)} IS NOT NULL`, + `SELECT ${k} AS k, ${hsql.q(ORG, anchor.holderObject)} AS org FROM ${t} WHERE ${k} IN (${chunk.map(() => '?').join(', ')}) AND ${hsql.q(ORG, anchor.holderObject)} IS NOT NULL`, chunk, ); for (const row of found) { @@ -890,6 +970,7 @@ async function planPresentTable(ctx: PlanContext, entry: InventoryEntry, source: const hasOrg = columns.has(ORG); const plan: TablePlan = { object: entry.object, + ...(ctx.database !== PRIMARY_DATABASE ? { database: ctx.database } : {}), fate: entry.fate, citation: entry.citation, source, @@ -1053,7 +1134,13 @@ async function walkNullRows( if (pending.length > 0 && ctx.posture === 'single' && ctx.defaultOrganizationId !== null) { derived['default-organization'] = (derived['default-organization'] ?? 0) + pending.length; for (const row of pending) { - assigned.push({ id: row.id, table: row.shard, organizationId: ctx.defaultOrganizationId, via: 'default-organization' }); + assigned.push({ + id: row.id, + table: row.shard, + ...(ctx.database !== PRIMARY_DATABASE ? { database: ctx.database } : {}), + organizationId: ctx.defaultOrganizationId, + via: 'default-organization', + }); } pending = []; } @@ -1103,6 +1190,18 @@ export async function buildOrganizationOwnershipPlan(opts: { reader: PlanReader; posture: string; database: string; + /** + * The databases beside the primary the deployment keeps objects in (the + * `telemetry` sibling), each read through its own reader. An object is + * planned in the database {@link homeOf} names for it. + */ + siblings?: readonly PlanSibling[]; + /** + * The database (`primary`, or a {@link PlanSibling.name}) the deployment + * keeps `object` in — the engine's routing on the booted stack — or + * `undefined` when the boot does not say (an object nothing declares). + */ + homeOf?: (object: string) => string | undefined; now?: Date; inventory?: readonly InventoryEntry[]; /** @@ -1112,11 +1211,21 @@ export async function buildOrganizationOwnershipPlan(opts: { onAttribution?: AttributionSink; }): Promise { const inventory = opts.inventory ?? ORGANIZATION_OWNERSHIP_INVENTORY; - // The reader's dialect was resolved from the driver before this call + const byObject = new Map(inventory.map((e) => [e.object, e])); + const inventoried = new Set(byObject.keys()); + // Each reader's dialect was resolved from its driver before this call // (`ceremonyDialectOf`, which refuses an unresolved one). const sql = new Sql(opts.reader, opts.reader.dialect); - const byObject = new Map(inventory.map((e) => [e.object, e])); - const catalog = await readCatalog(sql, opts.reader, new Set(byObject.keys())); + const databases: PlanDatabase[] = [{ name: PRIMARY_DATABASE, sql, catalog: await readCatalog(sql, opts.reader, inventoried) }]; + for (const sibling of opts.siblings ?? []) { + const siblingSql = new Sql(sibling.reader, sibling.reader.dialect); + databases.push({ name: sibling.name, sql: siblingSql, catalog: await readCatalog(siblingSql, sibling.reader, inventoried, true) }); + } + const catalog = databases[0]!.catalog; + const homeOf = (object: string): string => { + const home = opts.homeOf?.(object); + return home && databases.some((d) => d.name === home) ? home : PRIMARY_DATABASE; + }; // No sys_organization table (a deployment composed without the auth family) // holds no organization at all: nothing derives to a Default Organization. @@ -1133,43 +1242,97 @@ export async function buildOrganizationOwnershipPlan(opts: { ); defaultOrganizationId = rows.length > 0 ? nonEmpty(rows[0]!.id) : null; } - const ctx: PlanContext = { - sql, - catalog, - posture: opts.posture, - defaultOrganizationId, - ...(opts.onAttribution ? { onAttribution: opts.onAttribution } : {}), - }; const tables: TablePlan[] = []; + const staleCopies: StaleCopy[] = []; + const planned = new Set(); let outsideCeremony = 0; - for (const [base, physical] of [...catalog.bases].sort(([a], [b]) => a.localeCompare(b))) { - const entry = byObject.get(base); - const hasOrg = physical.some((t) => catalog.columns.get(t)?.has(ORG)); - if (!entry) { - if (!hasOrg) { + for (const db of databases) { + const ctx: PlanContext = { + database: db.name, + sql: db.sql, + catalog: db.catalog, + databases, + homeOf, + posture: opts.posture, + defaultOrganizationId, + ...(opts.onAttribution ? { onAttribution: opts.onAttribution } : {}), + }; + for (const [base, physical] of [...db.catalog.bases].sort(([a], [b]) => a.localeCompare(b))) { + const entry = byObject.get(base); + const hasOrg = physical.some((t) => db.catalog.columns.get(t)?.has(ORG)); + if (!hasOrg && !entry) { outsideCeremony++; continue; } - if (hasPlatformObjectPrefix(base)) { - throw new OrganizationOwnershipPlanRefusal( - 'uninventoried-platform-table', - `${base} carries ${ORG} but the ADR-0131 D10 inventory gives it no fate. The plan refuses rather than ` + - 'guess one: the inventory (packages/cli/src/utils/organization-ownership-inventory.ts) owes it a row, or, ' + - 'when the host provides the object, the host declares its fate in a fate supplement passed with ' + - '--fate-supplement FILE.', - base, - ); + // A table in a database its object does not live in, beside the one it + // does: a boot from before the object moved left it. Nothing reads it, so + // no fate is performed on it — it is listed and warned about, never + // attributed, deleted or dropped. (With no table where the object lives, + // the one found here is the object's table and is planned where it is.) + const home = homeOf(base); + if (home !== db.name && databases.find((d) => d.name === home)?.catalog.bases.has(base)) { + if (hasOrg) { + let rows = 0; + for (const t of physical) rows += await db.sql.count(t, `SELECT COUNT(*) AS n FROM ${db.sql.q(t, t)}`); + staleCopies.push({ object: base, database: db.name, home, tables: [...physical], rows }); + } else { + outsideCeremony++; + } + continue; } - tables.push(await planPresentTable(ctx, { object: base, census: [], ...APPLICATION_OBJECT_FATE }, 'application-default', physical)); - continue; + if (!entry) { + if (hasPlatformObjectPrefix(base)) { + throw new OrganizationOwnershipPlanRefusal( + 'uninventoried-platform-table', + `${base} carries ${ORG} but the ADR-0131 D10 inventory gives it no fate. The plan refuses rather than ` + + 'guess one: the inventory (packages/cli/src/utils/organization-ownership-inventory.ts) owes it a row, or, ' + + 'when the host provides the object, the host declares its fate in a fate supplement passed with ' + + '--fate-supplement FILE.', + base, + ); + } + tables.push(await planPresentTable(ctx, { object: base, census: [], ...APPLICATION_OBJECT_FATE }, 'application-default', physical)); + planned.add(base); + continue; + } + tables.push(await planPresentTable(ctx, entry, 'inventory', physical)); + planned.add(base); } - tables.push(await planPresentTable(ctx, entry, 'inventory', physical)); } for (const entry of inventory) { - if (!catalog.bases.has(entry.object)) tables.push(absentTable(entry)); + if (!planned.has(entry.object)) tables.push(absentTable(entry)); + } + tables.sort((a, b) => a.object.localeCompare(b.object) || (a.database ?? '').localeCompare(b.database ?? '')); + staleCopies.sort((a, b) => a.object.localeCompare(b.object)); + + const warnings: string[] = []; + const legacySets = tables.find((t) => t.object === LEGACY_SET_OBJECT && t.physical === 'present') + ?.categories.find((c) => c.id === LEGACY_SHIPPED_SET_SEEDS_CATEGORY); + if (legacySets && legacySets.rows > 0) { + const setColumns = catalog.columns.get(LEGACY_SET_OBJECT)!; + const where = predicateSql(sql, LEGACY_SET_OBJECT, setColumns, LEGACY_SHIPPED_SET_SEEDS); + const names = (await sql.read( + LEGACY_SET_OBJECT, + `SELECT ${sql.q('name', LEGACY_SET_OBJECT)} AS name FROM ${sql.q(LEGACY_SET_OBJECT, LEGACY_SET_OBJECT)} WHERE ${where.text}`, + where.params, + )).map((r) => String(r.name)); + warnings.push( + `${names.length} shipped permission set row(s) are stored as managed_by 'admin' (${[...new Set(names)].sort().join(', ')}): ` + + 'the seeder of an install created before 2026-08-15 wrote them that way. The ceremony adopts them as the platform ' + + 'seeds they are and deletes them as mirrors once every grant names its set, so the shipped definitions stand. ' + + 'An edit an administrator made to one of these sets in Setup is not carried: it stays readable in the backup ' + + 'taken before the upgrade, to author again as a new permission set.', + ); + } + for (const copy of staleCopies) { + warnings.push( + `${copy.object} has a table in the ${copy.database} database, but this deployment keeps the object in the ` + + `${copy.home} database: the ${copy.database} copy (${copy.rows} row(s)) is left over from an older boot and ` + + 'nothing reads it. The ceremony neither attributes nor drops it; drop it yourself once its rows are no ' + + 'longer needed.', + ); } - tables.sort((a, b) => a.object.localeCompare(b.object)); const byFate = { 'column-drop': 0, 'mirror-deletion': 0, attribution: 0, report: 0 } as Record; for (const t of tables) if (t.physical === 'present') byFate[t.fate]++; @@ -1186,6 +1349,9 @@ export async function buildOrganizationOwnershipPlan(opts: { defaultOrganization: defaultOrganizationId === null ? null : { id: defaultOrganizationId, slug: 'default' }, organizations, inventoryDigest: inventoryDigest(inventory), + ...(opts.siblings?.length ? { siblings: opts.siblings.map((d) => ({ name: d.name, database: d.database })) } : {}), + staleCopies, + warnings, tables, summary: { present: present.length, diff --git a/packages/cli/src/utils/schema-migration-plugins.ts b/packages/cli/src/utils/schema-migration-plugins.ts index f4cb0c5a2eb..ec2354f89e8 100644 --- a/packages/cli/src/utils/schema-migration-plugins.ts +++ b/packages/cli/src/utils/schema-migration-plugins.ts @@ -1745,6 +1745,7 @@ export async function buildSchemaMigrationPlugins(opts: { composed: [...opts.basePlugins, ...plugins], artifactRequires: opts.servedPlatform.artifactRequires, packageRoot: hostConfigPath ? path.dirname(hostConfigPath) : cwd, + ...(opts.serveFlags ? { serveFlags: opts.serveFlags } : {}), }); // The auth family where `serve` registers it — ahead of the host plugins, // so a config's own instance of the same name supersedes it here as @@ -1949,6 +1950,13 @@ async function composeServedPlatform(input: { composed: readonly unknown[]; artifactRequires?: readonly string[]; packageRoot: string; + /** + * `serve`'s flags as the command was given them: `--dev` is a development + * boot for the auth secret's fallback, exactly as {@link composeAuthGatedSecurity} + * reads it — so `--dev` composes the auth family (and with it the audit + * plugin's `sys_audit_log`) where `os dev` does, without `NODE_ENV`. + */ + serveFlags?: { readonly dev?: boolean; readonly preset?: string }; }): Promise<{ authFamily: unknown[]; plugins: unknown[]; notes: string[] }> { const { CAPABILITY_PROVIDERS, providesCapability, resolveServedCapabilities } = await import('@objectstack/core'); const config = input.config ?? {}; @@ -1964,7 +1972,7 @@ async function composeServedPlatform(input: { const decision = resolvePlatformAuthComposition({ plugins: [...input.basePlugins, ...input.hostPlugins], tiers: resolveStackTiers({ declaredTiers, requires }), - secret: resolveAuthSecret({ isDev: isDevelopmentBoot(undefined) }), + secret: resolveAuthSecret({ isDev: isDevelopmentBoot(input.serveFlags?.dev) }), }); if (decision.composes) { const { createIdentityObjectsPlugin } = await import('@objectstack/plugin-auth'); diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index fba777003d5..bf5c7df414b 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -76,6 +76,7 @@ export { isOrganizationOwnershipBootRefusal, organizationOwnershipRefusalMessage, } from './organization-ownership-boot-guard.js'; +export type { OrganizationOwnershipBootGuardOptions } from './organization-ownership-boot-guard.js'; export { DefaultDatasourcePlugin } from './default-datasource-plugin.js'; export type { DefaultDatasourceDefinition, DefaultDatasourcePluginOptions } from './default-datasource-plugin.js'; export { AppPlugin, collectBundleHooks, collectBundleFunctions, collectBundleFunctionEntries, collectBundleActions } from './app-plugin.js'; diff --git a/packages/runtime/src/organization-ownership-boot-guard.test.ts b/packages/runtime/src/organization-ownership-boot-guard.test.ts index 565fdf93cb2..aaf9be2afb3 100644 --- a/packages/runtime/src/organization-ownership-boot-guard.test.ts +++ b/packages/runtime/src/organization-ownership-boot-guard.test.ts @@ -90,10 +90,22 @@ describe('the v18 boot refusal (ADR-0131 D10 item 5)', () => { expect(err.code).toBe(ORGANIZATION_OWNERSHIP_BOOT_REFUSAL_CODE); expect(err.tables).toEqual(['sys_job', 'sys_metadata']); expect(err.message).toMatch(/still in the v17 organization-ownership shape: sys_job, sys_metadata/); - expect(err.message).toMatch(/os migrate organization-ownership --apply/); + // The apply the refusal names is one the ceremony accepts: a reviewed plan file and the backup acknowledged. + expect(err.message).toMatch(/os migrate organization-ownership --apply --plan-file PLAN --backup-taken/); expect(err.message).toMatch(/the ceremony has not run on it/); }); + it('a development boot names the ceremony with --dev, so it boots the way the refused boot did', async () => { + v17Shape(); + const dev = await refusalOf(() => assertOrganizationOwnershipMigrated(engine(), undefined, { dev: true })); + expect(dev.message).toContain( + "Run 'os migrate organization-ownership --dev --out PLAN' to write and review its plan, take a backup, then run " + + "'os migrate organization-ownership --dev --apply --plan-file PLAN --backup-taken'", + ); + const served = await refusalOf(() => assertOrganizationOwnershipMigrated(engine())); + expect(served.message).not.toContain('--dev'); + }); + it('a marker that is not verified (an unfinished ceremony, or one whose post-check found discrepancies) is refused', async () => { v17Shape(); marker(null); diff --git a/packages/runtime/src/organization-ownership-boot-guard.ts b/packages/runtime/src/organization-ownership-boot-guard.ts index db41037901a..5b0044cce8d 100644 --- a/packages/runtime/src/organization-ownership-boot-guard.ts +++ b/packages/runtime/src/organization-ownership-boot-guard.ts @@ -58,7 +58,23 @@ import { export const ORGANIZATION_OWNERSHIP_BOOT_REFUSAL_CODE = 'ORGANIZATION_OWNERSHIP_MIGRATION_REQUIRED' as const; const ORG = 'organization_id'; -const CEREMONY = 'os migrate organization-ownership'; + +/** + * The ceremony command the refusal names, spelled for the boot that refused: + * a development boot (`os dev`, `os serve --dev`) composes its security + * plugin, `.env*` files and `telemetry` sibling as a development boot, and the + * ceremony has to boot the same way to see the same database, so the command + * it names carries `--dev`. + */ +function ceremonyCommand(dev: boolean | undefined): string { + return dev ? 'os migrate organization-ownership --dev' : 'os migrate organization-ownership'; +} + +/** How the boot that runs the guard was started — what the refusal's command must match. */ +export interface OrganizationOwnershipBootGuardOptions { + /** A development boot (`os dev` / `os serve --dev`): the named ceremony command carries `--dev`. */ + dev?: boolean; +} export class OrganizationOwnershipBootRefusal extends Error { readonly code = ORGANIZATION_OWNERSHIP_BOOT_REFUSAL_CODE; @@ -161,7 +177,12 @@ export async function findV17ShapeTables(engine: GuardEngine): Promise } /** The refusal text: what was found, that the server refuses to start, the command, and the 17.x way out. */ -export function organizationOwnershipRefusalMessage(tables: readonly string[], marker: 'absent' | 'unverified' | 'unreadable'): string { +export function organizationOwnershipRefusalMessage( + tables: readonly string[], + marker: 'absent' | 'unverified' | 'unreadable', + opts: OrganizationOwnershipBootGuardOptions = {}, +): string { + const ceremony = ceremonyCommand(opts.dev); const state = marker === 'absent' ? 'the ceremony has not run on it' : marker === 'unverified' @@ -170,8 +191,9 @@ export function organizationOwnershipRefusalMessage(tables: readonly string[], m return ( `This database is still in the v17 organization-ownership shape: ${tables.join(', ')} still carr` + `${tables.length === 1 ? 'ies' : 'y'} a physical organization_id column, and ${state}. The server refuses to ` + - `start rather than serve v17 data with v18 semantics. Run '${CEREMONY}' (review its plan), then ` + - `'${CEREMONY} --apply', and start again. To keep 17.x semantics, run a 17.x runtime.` + `start rather than serve v17 data with v18 semantics. Run '${ceremony} --out PLAN' to write and review its ` + + `plan, take a backup, then run '${ceremony} --apply --plan-file PLAN --backup-taken', and start again. ` + + 'To keep 17.x semantics, run a 17.x runtime.' ); } @@ -183,6 +205,8 @@ export class OrganizationOwnershipBootGuardPlugin implements Plugin { /** Needs the engine to read columns and the marker; still loads (inert) without it. */ readonly optionalDependencies: string[] = ['com.objectstack.engine.objectql']; + constructor(private readonly options: OrganizationOwnershipBootGuardOptions = {}) {} + async init(_ctx: PluginContext): Promise {} async start(ctx: PluginContext): Promise { @@ -194,7 +218,7 @@ export class OrganizationOwnershipBootGuardPlugin implements Plugin { return; // no engine — no database to judge } if (!engine || typeof engine.getDriverForObject !== 'function') return; - await assertOrganizationOwnershipMigrated(engine, (ctx as any)?.logger); + await assertOrganizationOwnershipMigrated(engine, (ctx as any)?.logger, this.options); }); } } @@ -207,7 +231,9 @@ export class OrganizationOwnershipBootGuardPlugin implements Plugin { export async function assertOrganizationOwnershipMigrated( engine: GuardEngine, logger?: { info?: (msg: string) => void }, + opts: OrganizationOwnershipBootGuardOptions = {}, ): Promise { + const ceremony = ceremonyCommand(opts.dev); let tables: string[]; try { tables = await findV17ShapeTables(engine); @@ -215,7 +241,8 @@ export async function assertOrganizationOwnershipMigrated( throw new OrganizationOwnershipBootRefusal( `The server refuses to start: it could not read whether this database is still in the v17 organization-ownership ` + `shape (${error instanceof Error ? error.message : String(error)}), and an unread answer is never taken for a ` + - `migrated database. Fix the read, or run '${CEREMONY} --apply', then start again.`, + `migrated database. Fix the read, or run the ceremony ('${ceremony} --out PLAN', then ` + + `'${ceremony} --apply --plan-file PLAN --backup-taken'), then start again.`, [], ); } @@ -242,5 +269,5 @@ export async function assertOrganizationOwnershipMigrated( } catch { marker = 'unreadable'; } - throw new OrganizationOwnershipBootRefusal(organizationOwnershipRefusalMessage(tables, marker), tables); + throw new OrganizationOwnershipBootRefusal(organizationOwnershipRefusalMessage(tables, marker, opts), tables); }