From 12d55be3d415ecd8fe8be0fea82d9f2cc2a7c29c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 15:07:49 +0000 Subject: [PATCH 01/12] wip(plugin-audit): the mail-log body runner drives an archive datasource's driver too Claude-Session: https://claude.ai/code/session_01WQWCrorWxW5zDh4n6jCEF5 Co-authored-by: Claude --- packages/plugins/plugin-audit/src/index.ts | 9 +- .../src/mail-log-body-migration.ts | 134 +++++++++++++++--- 2 files changed, 120 insertions(+), 23 deletions(-) diff --git a/packages/plugins/plugin-audit/src/index.ts b/packages/plugins/plugin-audit/src/index.ts index 1a5c316a817..89ed6883244 100644 --- a/packages/plugins/plugin-audit/src/index.ts +++ b/packages/plugins/plugin-audit/src/index.ts @@ -75,12 +75,19 @@ export type { export { maskMailLogBodyCopy } from './mail-log-body-mask.js'; export { migrateMailLogBodyCopies, + migrateArchivedMailLogBodyCopies, planMailLogBodyAuditRowPatch, planMailLogBodyActivityRowPatch, MAIL_LOG_BODY_COPY_OBJECTS, MAIL_LOG_BODY_MIGRATION_PAGE_SIZE, } from './mail-log-body-migration.js'; -export type { MailLogBodyMigrationEngine, MailLogBodyMigrationReport } from './mail-log-body-migration.js'; +export type { + MailLogBodyArchiveDriver, + MailLogBodyCopyObject, + MailLogBodyMigrationEngine, + MailLogBodyMigrationOptions, + MailLogBodyMigrationReport, +} from './mail-log-body-migration.js'; // [ADR-0139 E6] The stored-row backfill of `sys_activity`'s two writer-stamped // read-rule columns, and the one planner it, the CRUD mirror and the // before-insert hook stamp by. The CLI command diff --git a/packages/plugins/plugin-audit/src/mail-log-body-migration.ts b/packages/plugins/plugin-audit/src/mail-log-body-migration.ts index 51e8b38cdad..cbbcac1458f 100644 --- a/packages/plugins/plugin-audit/src/mail-log-body-migration.ts +++ b/packages/plugins/plugin-audit/src/mail-log-body-migration.ts @@ -41,9 +41,24 @@ * `os migrate audit-metadata-bodies` takes). No `sys_migration` flag is * recorded: nothing gates irreversible behaviour on it. The report holds counts * only; no log line names a value. + * + * ## The archive, through the same runner + * + * `sys_audit_log` declares `lifecycle.archive`: the `LifecycleService` + * Archiver copies its rows past the hot window to the archive datasource + * VERBATIM (a driver `find` → `upsert`, no writer and no mask in between), + * then deletes them hot. So a copy archived before the writer masked keeps the + * body in clear for as long as the archive keeps it, and the engine never + * routes `sys_audit_log` there: an engine-driven run cannot reach it. + * {@link migrateArchivedMailLogBodyCopies} drives the very same pages and + * planners through the archive datasource's DRIVER instead. A hot copy masked + * before the Archiver moves it is archived masked, so the hot run and the + * archive run together cover the copies already archived and the ones still + * to be. */ -import type { IDataEngine } from '@objectstack/spec/contracts'; +import type { IDataDriver, IDataEngine } from '@objectstack/spec/contracts'; +import { isMissingTableError } from '@objectstack/types'; import { MAIL_LOG_OBJECTS } from '@objectstack/platform-objects/audit'; import { maskMailLogBodyCopy } from './mail-log-body-mask.js'; import type { MigrationLogger } from './stored-metadata-body-migration.js'; @@ -51,6 +66,9 @@ import type { MigrationLogger } from './stored-metadata-body-migration.js'; /** The two audit-family tables the writer copies a mail-log row into. */ export const MAIL_LOG_BODY_COPY_OBJECTS = ['sys_audit_log', 'sys_activity'] as const; +/** One of {@link MAIL_LOG_BODY_COPY_OBJECTS}. */ +export type MailLogBodyCopyObject = (typeof MAIL_LOG_BODY_COPY_OBJECTS)[number]; + /** Rows read per page. */ export const MAIL_LOG_BODY_MIGRATION_PAGE_SIZE = 500; @@ -128,6 +146,26 @@ export function planMailLogBodyActivityRowPatch(row: Record): { /** The engine surface the runner needs — the real data-engine contract's members. */ export type MailLogBodyMigrationEngine = Pick; +/** The driver surface the archive run needs — the real data-driver contract's members. */ +export type MailLogBodyArchiveDriver = Pick; + +/** What the runner options accept, for either store. */ +export interface MailLogBodyMigrationOptions { + apply?: boolean; + pageSize?: number; + /** The copy tables to walk — both by default. */ + objects?: readonly MailLogBodyCopyObject[]; +} + +/** + * The two operations a run performs, whatever store it runs on: read one page + * of `object`, and write one row's patch keeping its stored `updated_at`. + */ +interface CopyStore { + readPage(object: string, query: { where: Record; fields: string[]; orderBy: Array<{ field: string; order: 'asc' }>; limit: number }): Promise; + write(object: string, id: string, patch: Record, updatedAt: unknown): Promise; +} + /** What one migration run did (or would do in dry-run). Counts only. */ export interface MailLogBodyMigrationReport { apply: boolean; @@ -157,20 +195,85 @@ function asArray(result: unknown): Record[] { export async function migrateMailLogBodyCopies( engine: MailLogBodyMigrationEngine, logger: MigrationLogger, - options: { apply?: boolean; pageSize?: number } = {}, + options: MailLogBodyMigrationOptions = {}, +): Promise { + return runMailLogBodyCopies( + { + readPage: (object, query) => engine.find(object, query, { context: SYSTEM_CTX }), + // A single-id write, as the platform: a system caller is exempt from the + // static-readonly strip every audit/activity field carries. + // `preserveAudit` with the stored `updated_at` keeps the row's own + // timestamp (the shape `comment-parent-pair-backfill.ts` takes): masking + // a copy is not an edit of the ledger row. + write: async (object, id, patch, updatedAt) => { + await engine.update( + object, + { ...patch, id, ...(updatedAt !== undefined && updatedAt !== null ? { updated_at: updatedAt } : {}) }, + { context: SYSTEM_PRESERVING_CTX }, + ); + }, + }, + logger, + options, + 'audit/activity', + ); +} + +/** + * The same mask over an ARCHIVE datasource — the store `sys_audit_log`'s + * `lifecycle.archive.to` names, which the Archiver fills with verbatim copies + * and the engine never routes the object to. `driver` is that datasource's + * registered driver (`engine.datasource(name)`); the run reads and writes + * through it directly, as the Archiver itself does: no hook, no audit write, + * and the row's stored `updated_at` kept (`preserveAudit`). An archive table + * that was never created (nothing archived yet) holds no copy and reads as no + * rows; any other refused read is a failure, as on the hot run. + */ +export async function migrateArchivedMailLogBodyCopies( + driver: MailLogBodyArchiveDriver, + logger: MigrationLogger, + options: MailLogBodyMigrationOptions = {}, +): Promise { + return runMailLogBodyCopies( + { + readPage: async (object, query) => { + try { + return await driver.find(object, query as Parameters[1]); + } catch (e) { + if (isMissingTableError(e, object)) return []; + throw e; + } + }, + write: async (object, id, patch, updatedAt) => { + const keep = updatedAt !== undefined && updatedAt !== null; + await driver.update(object, id, { ...patch, ...(keep ? { updated_at: updatedAt } : {}) }, keep ? { preserveAudit: true } : undefined); + }, + }, + logger, + options, + 'archived audit', + ); +} + +async function runMailLogBodyCopies( + store: CopyStore, + logger: MigrationLogger, + options: MailLogBodyMigrationOptions, + label: string, ): Promise { const apply = options.apply === true; const pageSize = options.pageSize && options.pageSize > 0 ? options.pageSize : MAIL_LOG_BODY_MIGRATION_PAGE_SIZE; + const only = options.objects ? new Set(options.objects) : undefined; const report: MailLogBodyMigrationReport = { apply, byObject: {}, scanned: 0, rewritten: 0, failures: 0 }; const tables: ReadonlyArray<{ - object: (typeof MAIL_LOG_BODY_COPY_OBJECTS)[number]; + object: MailLogBodyCopyObject; fields: readonly string[]; plan: (row: Record) => Record | null; }> = [ - { object: 'sys_audit_log', fields: ['id', 'object_name', 'new_value', 'old_value', 'updated_at'], plan: planMailLogBodyAuditRowPatch }, - { object: 'sys_activity', fields: ['id', 'object_name', 'metadata', 'updated_at'], plan: planMailLogBodyActivityRowPatch }, - ]; + { object: 'sys_audit_log' as const, fields: ['id', 'object_name', 'new_value', 'old_value', 'updated_at'], plan: planMailLogBodyAuditRowPatch }, + { object: 'sys_activity' as const, fields: ['id', 'object_name', 'metadata', 'updated_at'], plan: planMailLogBodyActivityRowPatch }, + ].filter((t) => !only || only.has(t.object)); for (const { object, fields, plan } of tables) { const stats = (report.byObject[object] = { scanned: 0, rewritten: 0 }); @@ -183,11 +286,7 @@ export async function migrateMailLogBodyCopies( let rows: Record[]; try { rows = asArray( - await engine.find( - object, - { where, fields: [...fields], orderBy: [{ field: 'id', order: 'asc' }], limit: pageSize }, - { context: SYSTEM_CTX }, - ), + await store.readPage(object, { where, fields: [...fields], orderBy: [{ field: 'id', order: 'asc' }], limit: pageSize }), ); } catch (e) { // A table this run could not read is NOT a clean table: counted, so the @@ -216,16 +315,7 @@ export async function migrateMailLogBodyCopies( continue; } try { - // A single-id write, as the platform: a system caller is exempt from - // the static-readonly strip every audit/activity field carries. - // `preserveAudit` with the stored `updated_at` keeps the row's own - // timestamp (the shape `comment-parent-pair-backfill.ts` takes): - // masking a copy is not an edit of the ledger row. - await engine.update( - object, - { ...patch, id, ...(row.updated_at !== undefined && row.updated_at !== null ? { updated_at: row.updated_at } : {}) }, - { context: SYSTEM_PRESERVING_CTX }, - ); + await store.write(object, id, patch, row.updated_at); } catch (e) { report.failures += 1; logger.warn( @@ -242,7 +332,7 @@ export async function migrateMailLogBodyCopies( logger.info?.( `[mail-log-body-migration] ${apply ? 'masked' : 'would mask'} the mail body in ${report.rewritten} of ` + - `${report.scanned} audit/activity cop(ies) of a mail-log row` + + `${report.scanned} ${label} cop(ies) of a mail-log row` + (report.failures > 0 ? ` (${report.failures} failed — re-run to finish)` : ''), ); return report; From 136088a86cd0ff17b91f152a92a55da0bc531cce Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 15:21:00 +0000 Subject: [PATCH 02/12] wip(cli): os migrate audit-mail-bodies masks the archive datasource too Claude-Session: https://claude.ai/code/session_01WQWCrorWxW5zDh4n6jCEF5 Co-authored-by: Claude --- .../src/commands/migrate/audit-mail-bodies.ts | 143 ++++++++++++++++-- .../src/mail-log-body-migration.test.ts | 90 +++++++++++ 2 files changed, 219 insertions(+), 14 deletions(-) diff --git a/packages/cli/src/commands/migrate/audit-mail-bodies.ts b/packages/cli/src/commands/migrate/audit-mail-bodies.ts index eeaf91cb9be..ccdfa17fe38 100644 --- a/packages/cli/src/commands/migrate/audit-mail-bodies.ts +++ b/packages/cli/src/commands/migrate/audit-mail-bodies.ts @@ -4,6 +4,7 @@ import { Command, Flags } from '@oclif/core'; import chalk from 'chalk'; import { createInterface } from 'node:readline'; import type { IObjectQLEngine } from '@objectstack/spec/contracts'; +import type { MailLogBodyMigrationReport } from '@objectstack/plugin-audit'; import { printHeader, printSuccess, @@ -22,6 +23,43 @@ import { describeOccupancy, JOURNAL_MODE_SWITCH_NOTE } from '../../utils/sqlite- import { buildDataMigrationPlugins } from '../../utils/data-migration-plugins.js'; import { absentTableReads } from '../../utils/absent-table-reads.js'; +/** + * [#24899] What the command did about one declared archive target: `examined` + * (its driver is registered on this boot, and `report` is the run), `none` (no + * connected datasource and no runtime row of that name — the deployment has no + * archive), `not-connected` (an active runtime datasource row of that name + * exists, which this boot does not connect: NOT examined) or `unknown` (not + * connected, and that row could not be read). The last two fail the run. + */ +interface ArchiveOutcome { + datasource: string; + objects: string[]; + status: 'examined' | 'none' | 'not-connected' | 'unknown'; + report?: MailLogBodyMigrationReport; +} + +/** + * Does the database hold an active runtime datasource row named `datasource`? + * Runtime datasources are `sys_metadata` rows of type `datasource` — the + * store `os serve`'s datasource admin rehydrates its pools from at boot. Ids + * only are read; a refused read answers `unknown`, never `none`. + */ +async function runtimeDatasourceRowStatus( + engine: Pick, + datasource: string, +): Promise<'none' | 'not-connected' | 'unknown'> { + try { + const rows = await engine.find( + 'sys_metadata', + { where: { type: 'datasource', name: datasource, state: 'active' }, fields: ['id'], limit: 1 }, + { context: { isSystem: true } }, + ); + return Array.isArray(rows) && rows.length > 0 ? 'not-connected' : 'none'; + } catch { + return 'unknown'; + } +} + async function confirm(question: string): Promise { if (!process.stdin.isTTY) return false; // non-interactive → require --yes const rl = createInterface({ input: process.stdin, output: process.stdout }); @@ -57,12 +95,32 @@ async function confirm(question: string): Promise { * clean report is the verification. The report is counts only. No * `sys_migration` flag is recorded: nothing gates irreversible behaviour on it. * The flags are `os migrate audit-metadata-bodies`'s, one for one. + * + * ## The archive datasource too (#24899) + * + * `sys_audit_log` declares `lifecycle.archive` (`to: 'archive'`): the + * Archiver copies rows past the hot window to that datasource verbatim and + * deletes them hot, and the engine never routes the object there. So a copy + * archived before the writer masked still holds the body, out of the hot run's + * reach. After the hot run this command reads each copy table's declared + * archive target and, when this boot connected a datasource of that name, + * runs the same runner through its driver (`migrateArchivedMailLogBodyCopies`), + * in the same dry-run / `--apply` mode. A hot copy masked before the Archiver + * moves it is archived masked, so the two runs cover the copies already + * archived and the ones still to be. + * + * Absence is loud: when the target is not connected here but the database + * holds an active runtime datasource row of that name (one an admin created, + * which `os serve` connects and this boot does not), the archive was NOT + * examined — the report says so and the command exits non-zero. No such row + * and no connected datasource means the deployment has no archive, and the + * Archiver retains every row hot: nothing to mask there. */ export default class MigrateAuditMailBodies extends Command { static override description = 'Mask the mail bodies (body_text, body_html) of sys_email / sys_identity_email rows that the audit writer ' + - 'copied in clear into sys_audit_log / sys_activity before it masked them. Dry run by default; --apply writes. ' + - 'Reports counts only.'; + 'copied in clear into sys_audit_log / sys_activity before it masked them, on this database and in the archive ' + + 'datasource sys_audit_log archives to. Dry run by default; --apply writes. Reports counts only.'; static override examples = [ '$ os migrate audit-mail-bodies', @@ -125,12 +183,12 @@ export default class MigrateAuditMailBodies extends Command { this.exit(1); return; } - printWarning('Apply mode rewrites audit/activity rows. Re-run with --yes to confirm, or run without --apply to preview.'); + printWarning('Apply mode rewrites audit/activity rows (and archived audit rows). Re-run with --yes to confirm, or run without --apply to preview.'); this.exit(1); return; } const ok = await confirm( - chalk.bold('\nMask the mail body in every audit/activity copy of a mail-log row on this database? [y/N] '), + chalk.bold('\nMask the mail body in every audit/activity copy of a mail-log row on this database and its archive? [y/N] '), ); if (!ok) { printInfo('Aborted — no changes made.'); @@ -171,7 +229,8 @@ export default class MigrateAuditMailBodies extends Command { throw new Error('No ObjectQL engine on this stack — cannot rewrite audit rows.'); } - const { migrateMailLogBodyCopies } = await import('@objectstack/plugin-audit'); + const { migrateMailLogBodyCopies, migrateArchivedMailLogBodyCopies, MAIL_LOG_BODY_COPY_OBJECTS } = + await import('@objectstack/plugin-audit'); const logger = flags.json ? { info: (m: string) => console.error(m), warn: (m: string) => console.error(m) } @@ -192,10 +251,38 @@ export default class MigrateAuditMailBodies extends Command { const report = await migrateMailLogBodyCopies(readView, logger, { apply }); + // [#24899] Each copy table's declared archive target, then the same + // runner through that datasource's driver — the registry the Archiver + // itself resolves `archive.to` against (`engine.datasource(name)`). + const targets = new Map>(); + for (const object of MAIL_LOG_BODY_COPY_OBJECTS) { + const to = (engine.getObject(object) as { lifecycle?: { archive?: { to?: unknown } } } | undefined) + ?.lifecycle?.archive?.to; + if (typeof to !== 'string' || to === '') continue; + targets.set(to, [...(targets.get(to) ?? []), object]); + } + const archive: ArchiveOutcome[] = []; + for (const [datasource, objects] of targets) { + const driver = engine.getDriverByName?.(datasource); + if (driver) { + const archived = await migrateArchivedMailLogBodyCopies(driver, logger, { apply, objects }); + archive.push({ datasource, objects, status: 'examined', report: archived }); + continue; + } + // Not connected on this boot. A runtime datasource row of that name is + // an archive `os serve` connects and this boot cannot: NOT examined. + archive.push({ datasource, objects, status: await runtimeDatasourceRowStatus(readView, datasource) }); + } + const archiveFailures = archive.reduce( + (n, a) => n + (a.report?.failures ?? 0) + (a.status === 'not-connected' || a.status === 'unknown' ? 1 : 0), + 0, + ); + const failures = report.failures + archiveFailures; + if (flags.json) { reads.notice(true); - await emitJson({ database: stack.dbLabel, apply, report, duration: timer.elapsed() }); - if (report.failures > 0) this.exit(1); + await emitJson({ database: stack.dbLabel, apply, report, archive, duration: timer.elapsed() }); + if (failures > 0) this.exit(1); return; } @@ -207,22 +294,50 @@ export default class MigrateAuditMailBodies extends Command { ` ${chalk.white(object)}: ${stats.scanned} mail-log cop(ies) scanned, ${stats.rewritten} ${apply ? 'masked' : 'to mask'}`, ); } + for (const a of archive) { + if (a.report) { + for (const [object, stats] of Object.entries(a.report.byObject)) { + console.log( + ` ${chalk.white(object)} (archive datasource '${a.datasource}'): ${stats.scanned} mail-log cop(ies) scanned, ` + + `${stats.rewritten} ${apply ? 'masked' : 'to mask'}`, + ); + } + } else if (a.status === 'none') { + console.log(chalk.dim(` archive datasource '${a.datasource}': none on this deployment — nothing archived, nothing to mask`)); + } + } console.log(''); + for (const a of archive) { + if (a.status === 'not-connected') { + printError( + `Archive datasource '${a.datasource}' (${a.objects.join(', ')}) was NOT examined: the database holds an active ` + + `runtime datasource of that name, which \`os serve\` connects and this command does not, so its archived ` + + `copies may still hold a mail body in clear. Declare it in the stack config's \`datasources\` ` + + `(with \`autoConnect: true\`) and re-run.`, + ); + } else if (a.status === 'unknown') { + printError( + `Archive datasource '${a.datasource}' (${a.objects.join(', ')}) was NOT examined: it is not connected on this ` + + `boot, and whether the database holds a runtime datasource of that name could not be read.`, + ); + } + } - if (report.failures > 0) { - printError(`${report.failures} read(s) or row(s) failed — re-run to finish them.`); - } else if (apply && report.rewritten > 0) { + const rewritten = report.rewritten + archive.reduce((n, a) => n + (a.report?.rewritten ?? 0), 0); + if (failures > 0) { + printError(`${failures} read(s), row(s) or archive datasource(s) failed — re-run to finish them.`); + } else if (apply && rewritten > 0) { printSuccess( - `Masked the mail body in ${report.rewritten} audit/activity row(s). Re-run any time — it only revisits copies still holding a body.`, + `Masked the mail body in ${rewritten} audit/activity row(s). Re-run any time — it only revisits copies still holding a body.`, ); - } else if (!apply && report.rewritten > 0) { - printInfo(`Dry run only — ${report.rewritten} row(s) would be masked. Re-run with --apply.`); + } else if (!apply && rewritten > 0) { + printInfo(`Dry run only — ${rewritten} row(s) would be masked. Re-run with --apply.`); } else { printSuccess('Nothing to mask — no audit/activity copy of a mail-log row holds a mail body in clear.'); } console.log(chalk.dim(` ${timer.display()}`)); console.log(''); - if (report.failures > 0) this.exit(1); + if (failures > 0) this.exit(1); } catch (error: any) { if (isExitSignal(error)) throw error; if (flags.json) { await emitJson({ error: error.message, ...errorCodeFields(error) }, 0, { compact: true }); this.exit(1); } diff --git a/packages/plugins/plugin-audit/src/mail-log-body-migration.test.ts b/packages/plugins/plugin-audit/src/mail-log-body-migration.test.ts index 04f777a4a84..75b7424ad4e 100644 --- a/packages/plugins/plugin-audit/src/mail-log-body-migration.test.ts +++ b/packages/plugins/plugin-audit/src/mail-log-body-migration.test.ts @@ -19,8 +19,11 @@ import { describe, expect, it, vi } from 'vitest'; // `stored-metadata-body-migration.test.ts` takes. import { assertEngineUpdateDispatch } from '@objectstack/metadata-core'; import { SECRET_MASK } from '@objectstack/objectql/core'; +import { SqliteWasmDriver } from '@objectstack/driver-sqlite-wasm'; import { maskMailLogBodyCopy } from './mail-log-body-mask.js'; +import { SysAuditLog } from './objects/sys-audit-log.object.js'; import { + migrateArchivedMailLogBodyCopies, migrateMailLogBodyCopies, planMailLogBodyActivityRowPatch, planMailLogBodyAuditRowPatch, @@ -265,3 +268,90 @@ describe('migrateMailLogBodyCopies (driven, paged)', () => { expect(lines).not.toMatch(/mail-body-sentinel/); }); }); + +describe('migrateArchivedMailLogBodyCopies (an archive datasource, through its driver)', () => { + // [#24899] The Archiver copies `sys_audit_log` rows to the archive + // datasource VERBATIM (driver `find` → `upsert`), and the engine never routes + // the object there. So the store here is a REAL SQL driver holding the table + // exactly as the Archiver provisions and fills it: `syncSchema` with the + // object's own declaration, then `upsert(object, row, ['id'])` per row. + async function archiveStore(rows: Row[]): Promise { + const cold = new SqliteWasmDriver({ filename: ':memory:' }); + await cold.connect(); + await cold.syncSchema('sys_audit_log', SysAuditLog); + for (const row of rows) await cold.upsert('sys_audit_log', row, ['id']); + return cold; + } + const allRows = async (cold: SqliteWasmDriver): Promise => + (await cold.find('sys_audit_log', { orderBy: [{ field: 'id', order: 'asc' }] })).map((r) => ({ ...r })); + const logger = () => ({ info: vi.fn(), warn: vi.fn() }); + const linesOf = (log: ReturnType): string => + [...log.info.mock.calls, ...log.warn.mock.calls].flat().join('\n'); + + it('a dry run counts the archived clear copies and writes nothing', async () => { + const cold = await archiveStore(auditRows()); + const before = await allRows(cold); + expect(clearBodies(before)).toBe(3); + const report = await migrateArchivedMailLogBodyCopies(cold, logger(), { objects: ['sys_audit_log'] }); + expect(report).toEqual({ + apply: false, + byObject: { sys_audit_log: { scanned: 4, rewritten: 3 } }, + scanned: 4, + rewritten: 3, + failures: 0, + }); + expect(isDeepStrictEqual(await allRows(cold), before)).toBe(true); + }); + + it('--apply leaves 0 archived clear copies, moves nothing else, and a re-run changes no byte', async () => { + const cold = await archiveStore(auditRows()); + const before = new Map((await allRows(cold)).map((r) => [r.id, r])); + const log = logger(); + const report = await migrateArchivedMailLogBodyCopies(cold, log, { apply: true, pageSize: 2, objects: ['sys_audit_log'] }); + expect(report.rewritten).toBe(3); + expect(report.failures).toBe(0); + + const after = await allRows(cold); + expect(clearBodies(after)).toBe(0); + for (const row of after) { + const original = before.get(row.id)!; + for (const key of Object.keys(original)) { + if (key === 'new_value' || key === 'old_value') continue; + // `updated_at` included: the archived row keeps its own stamp. + expect(row[key], `${String(row.id)}.${key}`).toEqual(original[key]); + } + } + // The masked copy reads the writer's mask, and every other key it held is kept. + const created = JSON.parse(String(after.find((r) => r.id === 'al_1')!.new_value)); + expect(created.body_text).toBe(SECRET_MASK); + expect(created.body_html).toBe(SECRET_MASK); + expect(created.subject).toBe('Subject kept'); + // Another object's archived row is untouched. + expect(after.find((r) => r.id === 'al_5')).toEqual(before.get('al_5')); + // Counts only: no log line names a body. + expect(linesOf(log)).not.toMatch(/mail-body-sentinel/); + + const again = await migrateArchivedMailLogBodyCopies(cold, logger(), { apply: true, objects: ['sys_audit_log'] }); + expect(again.rewritten).toBe(0); + expect(again.scanned).toBe(4); + expect(isDeepStrictEqual(await allRows(cold), after)).toBe(true); + }); + + it('an archive table nothing was archived into yet reads as no rows, never a failure', async () => { + const cold = new SqliteWasmDriver({ filename: ':memory:' }); + await cold.connect(); + const report = await migrateArchivedMailLogBodyCopies(cold, logger(), { apply: true }); + expect(report).toMatchObject({ scanned: 0, rewritten: 0, failures: 0 }); + }); + + it('any other refused archive read is a failure, never a clean report', async () => { + const cold = await archiveStore(auditRows()); + const refusing = { + find: async () => { throw new Error('archive read refused'); }, + update: cold.update.bind(cold), + }; + const report = await migrateArchivedMailLogBodyCopies(refusing, logger(), { apply: true, objects: ['sys_audit_log'] }); + expect(report.failures).toBe(1); + expect(report.rewritten).toBe(0); + }); +}); From 009eedebc8ce5d8ef83fb95d9c36f197f1867542 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 15:25:56 +0000 Subject: [PATCH 03/12] wip(cli): pin the archive mask end to end Claude-Session: https://claude.ai/code/session_01WQWCrorWxW5zDh4n6jCEF5 Co-authored-by: Claude --- ...it-mail-bodies.archive.integration.test.ts | 344 ++++++++++++++++++ 1 file changed, 344 insertions(+) create mode 100644 packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts diff --git a/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts b/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts new file mode 100644 index 00000000000..2c9cecd9333 --- /dev/null +++ b/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts @@ -0,0 +1,344 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#24899] `os migrate audit-mail-bodies` masks the mail-log bodies the + * Archiver already moved to `sys_audit_log`'s archive datasource, and the hot + * copies it will move later are archived masked. + * + * ## The measured gap + * + * `sys_audit_log` declares `lifecycle.archive: { after: '90d', to: 'archive' }`. + * The Archiver (`LifecycleService`) copies rows past the hot window to the + * datasource named `archive` VERBATIM — a driver `find` → `upsert`, no writer + * and no mask in between — then deletes them hot, and the engine never routes + * `sys_audit_log` there. So a copy archived before the writer masked kept the + * body in clear out of the command's reach: it masked the hot engine only. + * + * ## The fixture + * + * A project declaring an `archive` SQLite datasource with `autoConnect: true` + * (ADR-0062 D2(c): a managed datasource nothing routes to connects only so), + * and a served-shape boot that seeds 18.0-writer copies of mail-log rows with + * the body in clear — two past the hot window, one inside it, one non-mail row + * past it — then runs the REAL Archiver (`lifecycle.sweep()`). The archive + * store is therefore exactly what a deployment's is. + * + * ## What is pinned + * + * 0. The Archiver moved the old copies to the archive with the body in clear + * (a vacuous fixture is no pin), and left the young one hot. + * 1. The dry run counts the hot copy and the archived ones, names the archive + * datasource `examined`, and writes nothing to either store. + * 2. `--apply` leaves 0 copies holding a body in clear in either store; every + * other archived row is untouched. + * 3. A second `--apply` masks nothing and changes no archived row. + * 4. The copy that was hot when the command ran, aged past the window and + * archived by a second sweep, lands in the archive masked. + * 5. No run prints a body. + * 6. A deployment whose archive is a RUNTIME datasource row (which `os serve` + * connects and this command's boot does not) is NOT reported clean: the + * archive reads `not-connected` and the command exits 1. With neither a + * connected datasource nor such a row it reads `none` and exits 0. + * + * ⛔ The bodies are sentinels, never a link or a token, and every comparison is + * reported as a count or a boolean. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawn, spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import Database from 'better-sqlite3'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const CLI_ROOT = resolve(HERE, '..', '..', '..'); +/** The source entry, as `test/helpers/serve-process.ts` spawns it; `src/` cannot import that helper. */ +const CLI = resolve(HERE, '../../../bin/run-dev.js'); + +const HOOK_TIMEOUT_MS = 480_000; +const RUN_BUDGET_MS = 120_000; +const SENTINEL = 'mail-body-sentinel-24899'; +const MASK = '••••••••'; + +/** Env that would point a boot somewhere other than the fixture. */ +const OVERRIDING_ENV = ['OS_DATABASE_URL', 'DATABASE_URL', 'TURSO_DATABASE_URL', 'OS_DATABASE_DRIVER', 'OS_HOME'] as const; + +/** + * This process's environment for a child, minus the families + * `test/helpers/serve-process.ts` `childEnv()` strips (its header says why). + */ +function childEnv(overrides: Record): Record { + const env: Record = {}; + for (const [key, value] of Object.entries(process.env)) { + if (key === 'TEST' || key === 'VITEST' || key.startsWith('VITEST_') || key === 'NODE_PATH') continue; + env[key] = value; + } + const unset: Record = {}; + for (const key of OVERRIDING_ENV) unset[key] = undefined; + return { + ...env, + ...unset, + // The fixed key `serve-process.ts` hands its children, so no boot mints and + // persists a crypto key into this runner's home directory. + OS_SECRET_KEY: '0e2e'.repeat(16), + // [#22579] One database for the hot objects: the fixture declares its own topology. + OS_TELEMETRY_DB: '0', + NO_COLOR: '1', + ...overrides, + }; +} + +/** + * A served-shape boot of the data stack plus the audit plugin, with the + * fixture's `archive` datasource. `MODE=seed` plants the 18.0-writer copies; + * `MODE=age` moves every hot copy's `created_at` past the window. Either way + * the boot then runs the real Archiver once. + */ +const SWEEP_CHILD = ` +const rt = await import('@objectstack/runtime'); +const { PlatformObjectsPlugin } = await import('@objectstack/platform-objects/plugin'); +const { AuditPlugin } = await import('@objectstack/plugin-audit'); +const { SqlDriver } = await import('@objectstack/driver-sql'); +const stack = await rt.createStandaloneStack({ + projectRoot: process.env.FIXTURE_PROJECT, + databaseUrl: 'file:' + process.env.FIXTURE_DB, + skipSeedData: true, + armLifecycleSweep: false, +}); +const runtime = new rt.Runtime({ cluster: false }); +const kernel = runtime.getKernel(); +for (const plugin of stack.plugins) await kernel.use(plugin); +await kernel.use(new PlatformObjectsPlugin()); +await kernel.use(new AuditPlugin()); +await runtime.start(); +const ql = kernel.getService('objectql'); +let source = 'declared'; +if (!ql.getDriverByName('archive')) { + // The host registers the archive driver under the datasource's name, as + // the datasource-connection service does for a connected declaration. + const cold = new SqlDriver({ client: 'better-sqlite3', connection: { filename: process.env.FIXTURE_ARCHIVE }, useNullAsDefault: true }); + Object.defineProperty(cold, 'name', { value: 'archive' }); + await cold.connect(); + ql.registerDriver(cold); + source = 'host-registered'; +} +const k = ql.getDriverForObject('sys_audit_log').knex; +const old = new Date(Date.now() - 200 * 86400000).toISOString(); +if (process.env.MODE === 'seed') { + const copy = (id) => JSON.stringify({ id, subject: 'Subject kept', status: 'sent', body_text: process.env.SENTINEL, body_html: process.env.SENTINEL }); + await k('sys_audit_log').insert([ + { id: 'aud_old_1', object_name: 'sys_email', record_id: 'em_1', action: 'create', created_at: old, updated_at: old, new_value: copy('em_1') }, + { id: 'aud_old_2', object_name: 'sys_identity_email', record_id: 'ie_1', action: 'delete', created_at: old, updated_at: old, old_value: copy('ie_1') }, + { id: 'aud_old_3', object_name: 'crm_note', record_id: 'n_1', action: 'create', created_at: old, updated_at: old, new_value: JSON.stringify({ id: 'n_1', title: 'kept' }) }, + { id: 'aud_young', object_name: 'sys_email', record_id: 'em_2', action: 'create', created_at: new Date().toISOString(), new_value: copy('em_2') }, + ]); +} else { + await k('sys_audit_log').update({ created_at: old }); +} +const report = await kernel.getService('lifecycle').sweep(); +const entry = report.swept.find((s) => s.object === 'sys_audit_log'); +await kernel.shutdown(); +process.stderr.write('[fixture] swept ' + JSON.stringify({ source, archived: entry ? entry.archived : null }) + '\\n'); +process.exit(0); +`; + +interface Run { + code: number | null; + stdout: string; + stderr: string; +} + +let dir: string; +let mainDb: string; +let archiveDb: string; +let runtimeRowDir: string; +let runtimeRowDb: string; +let seedDetail: { source: string; archived: number | null }; +let ageDetail: { source: string; archived: number | null }; +const runs: Record<'dry' | 'apply' | 'again', Run> = {} as never; +let archivedBefore: Record[]; +let archivedAfterApply: Record[]; +let archivedAfterAgain: Record[]; +let hotAfterApply: Record[]; +let archivedAfterAging: Record[]; +let runtimeRowRun: Run; +let noArchiveRun: Run; + +function sweep(mode: 'seed' | 'age'): { source: string; archived: number | null } { + const r = spawnSync(process.execPath, ['--input-type=module', '-e', SWEEP_CHILD], { + cwd: CLI_ROOT, + env: childEnv({ + OS_ARTIFACT_PATH: join(dir, 'dist', 'objectstack.json'), + FIXTURE_PROJECT: dir, + FIXTURE_DB: mainDb, + FIXTURE_ARCHIVE: archiveDb, + SENTINEL, + MODE: mode, + }), + encoding: 'utf8', + timeout: RUN_BUDGET_MS, + }); + const line = String(r.stderr).split('\n').find((l) => l.startsWith('[fixture] swept ')); + if (r.status !== 0 || !line) throw new Error(`the ${mode} sweep did not run (status ${r.status})\n${String(r.stderr).slice(-3000)}`); + return JSON.parse(line.slice('[fixture] swept '.length)); +} + +/** One `os migrate audit-mail-bodies --database-url file:` run, in `project`. */ +function runCommand(project: string, db: string, extra: string[]): Promise { + return new Promise((resolveRun, rejectRun) => { + const child = spawn( + process.execPath, + [CLI, 'migrate', 'audit-mail-bodies', ...extra, '--database-url', `file:${db}`], + { cwd: project, env: childEnv({ OS_ARTIFACT_PATH: join(project, 'dist', 'objectstack.json') }), stdio: ['ignore', 'pipe', 'pipe'] }, + ); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (c) => { stdout += String(c); }); + child.stderr.on('data', (c) => { stderr += String(c); }); + const timer = setTimeout(() => { + child.kill('SIGKILL'); + rejectRun(new Error(`os migrate audit-mail-bodies did not finish within ${RUN_BUDGET_MS}ms`)); + }, RUN_BUDGET_MS); + child.on('error', (err) => { clearTimeout(timer); rejectRun(err); }); + child.on('close', (code) => { clearTimeout(timer); resolveRun({ code, stdout, stderr }); }); + }); +} + +/** Every `sys_audit_log` row of one database file, read with a connection of our own. */ +function auditRows(file: string): Record[] { + const db = new Database(file, { readonly: true }); + try { + return db.prepare('SELECT * FROM sys_audit_log ORDER BY id').all() as Record[]; + } finally { + db.close(); + } +} + +/** Rows that still hold a sentinel body in clear. */ +const clear = (rows: Record[]): number => rows.filter((r) => JSON.stringify(r).includes(SENTINEL)).length; + +function writeArtifact(project: string, datasources: unknown[]): void { + mkdirSync(join(project, 'dist'), { recursive: true }); + writeFileSync( + join(project, 'dist', 'objectstack.json'), + JSON.stringify({ + manifest: { id: 'com.example.os24899', name: 'Archived mail bodies', version: '0.0.0', type: 'app' }, + datasources, + objects: [{ name: 'os24899_thing', fields: { name: { type: 'text' } } }], + }), + ); +} + +beforeAll(async () => { + dir = mkdtempSync(join(tmpdir(), 'os-24899-')); + mkdirSync(join(dir, 'data'), { recursive: true }); + mainDb = join(dir, 'data', 'main.db'); + archiveDb = join(dir, 'data', 'archive.db'); + writeArtifact(dir, [{ name: 'archive', driver: 'sqlite', config: { filename: archiveDb }, autoConnect: true }]); + + seedDetail = sweep('seed'); + archivedBefore = auditRows(archiveDb); + + runs.dry = await runCommand(dir, mainDb, ['--json']); + runs.apply = await runCommand(dir, mainDb, ['--apply', '--yes', '--json']); + archivedAfterApply = auditRows(archiveDb); + hotAfterApply = auditRows(mainDb); + runs.again = await runCommand(dir, mainDb, ['--apply', '--yes', '--json']); + archivedAfterAgain = auditRows(archiveDb); + + ageDetail = sweep('age'); + archivedAfterAging = auditRows(archiveDb); + + // A deployment whose archive is an admin-created runtime datasource: no + // declaration, and an active `datasource` row in `sys_metadata`. + runtimeRowDir = mkdtempSync(join(tmpdir(), 'os-24899-runtime-row-')); + runtimeRowDb = join(runtimeRowDir, 'main.db'); + writeArtifact(runtimeRowDir, []); + noArchiveRun = await runCommand(runtimeRowDir, runtimeRowDb, ['--apply', '--yes', '--json']); + const db = new Database(runtimeRowDb); + try { + db.prepare("INSERT INTO sys_metadata (id, name, type, scope, metadata, state, version) VALUES ('md_24899', 'archive', 'datasource', 'platform', ?, 'active', 1)") + .run(JSON.stringify({ name: 'archive', driver: 'sqlite', config: { filename: './archive.db' }, origin: 'runtime' })); + } finally { + db.close(); + } + runtimeRowRun = await runCommand(runtimeRowDir, runtimeRowDb, ['--json']); +}, HOOK_TIMEOUT_MS); + +afterAll(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + if (runtimeRowDir) rmSync(runtimeRowDir, { recursive: true, force: true }); +}); + +const doc = (run: Run): any => JSON.parse(run.stdout); +const archiveOf = (run: Run): any => doc(run).archive.find((a: { datasource: string }) => a.datasource === 'archive'); + +describe('[#24899] os migrate audit-mail-bodies over an archive datasource', () => { + it('0. the fixture: the real Archiver moved the old copies to the archive with the body in clear', () => { + expect(seedDetail.archived).toBe(3); + expect(archivedBefore.length).toBe(3); + expect(clear(archivedBefore)).toBe(2); + }); + + it('1. the dry run counts the hot and the archived copies, and writes nothing', () => { + const run = runs.dry; + expect(run.code, run.stderr.slice(-2000)).toBe(0); + expect(doc(run).report).toMatchObject({ failures: 0, byObject: { sys_audit_log: { rewritten: 1 } } }); + expect(archiveOf(run)).toMatchObject({ + status: 'examined', + objects: ['sys_audit_log'], + report: { apply: false, failures: 0, byObject: { sys_audit_log: { scanned: 2, rewritten: 2 } } }, + }); + // Written only by --apply below: the dry run left the archive as the Archiver did. + expect(clear(archivedBefore)).toBe(2); + }); + + it('2. --apply leaves 0 clear copies in either store, and every other archived row is untouched', () => { + const run = runs.apply; + expect(run.code, run.stderr.slice(-2000)).toBe(0); + expect(archiveOf(run).report).toMatchObject({ apply: true, rewritten: 2, failures: 0 }); + expect(clear(archivedAfterApply)).toBe(0); + expect(clear(hotAfterApply)).toBe(0); + const before = new Map(archivedBefore.map((r) => [r.id, r])); + for (const row of archivedAfterApply) { + const original = before.get(row.id)!; + for (const key of Object.keys(original)) { + if (key === 'new_value' || key === 'old_value') continue; + expect(row[key], `${String(row.id)}.${key}`).toEqual(original[key]); + } + } + expect(archivedAfterApply.find((r) => r.id === 'aud_old_3')).toEqual(before.get('aud_old_3')); + const masked = JSON.parse(String(archivedAfterApply.find((r) => r.id === 'aud_old_1')!.new_value)); + expect([masked.body_text === MASK, masked.body_html === MASK, masked.subject === 'Subject kept']).toEqual([true, true, true]); + }); + + it('3. a second --apply masks nothing and changes no archived row', () => { + const run = runs.again; + expect(run.code, run.stderr.slice(-2000)).toBe(0); + expect(doc(run).report.rewritten).toBe(0); + expect(archiveOf(run).report).toMatchObject({ scanned: 2, rewritten: 0, failures: 0 }); + expect(archivedAfterAgain).toEqual(archivedAfterApply); + }); + + it('4. the copy that was hot when the command ran is archived masked', () => { + expect(ageDetail.archived).toBe(1); + expect(archivedAfterAging.length).toBe(4); + expect(archivedAfterAging.some((r) => r.id === 'aud_young')).toBe(true); + expect(clear(archivedAfterAging)).toBe(0); + }); + + it('5. no run prints a body', () => { + for (const run of [runs.dry, runs.apply, runs.again, runtimeRowRun, noArchiveRun]) { + expect(`${run.stdout}\n${run.stderr}`.includes(SENTINEL)).toBe(false); + } + }); + + it('6. a runtime archive datasource this boot does not connect is NOT reported clean', () => { + expect(noArchiveRun.code, noArchiveRun.stderr.slice(-2000)).toBe(0); + expect(archiveOf(noArchiveRun)).toEqual({ datasource: 'archive', objects: ['sys_audit_log'], status: 'none' }); + expect(runtimeRowRun.code).toBe(1); + expect(archiveOf(runtimeRowRun)).toEqual({ datasource: 'archive', objects: ['sys_audit_log'], status: 'not-connected' }); + }); +}); From e302ca050125a45ba34fd465e518d270b954e89e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 15:32:56 +0000 Subject: [PATCH 04/12] wip(cli): connect declared datasources and end the process deliberately Claude-Session: https://claude.ai/code/session_01WQWCrorWxW5zDh4n6jCEF5 Co-authored-by: Claude --- ...udit-mail-bodies.archive.integration.test.ts | 3 ++- .../src/commands/migrate/audit-mail-bodies.ts | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts b/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts index 2c9cecd9333..ebad5bc68e5 100644 --- a/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts +++ b/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts @@ -105,7 +105,8 @@ const stack = await rt.createStandaloneStack({ projectRoot: process.env.FIXTURE_PROJECT, databaseUrl: 'file:' + process.env.FIXTURE_DB, skipSeedData: true, - armLifecycleSweep: false, + // Armed, so the service is enabled: the sweep below is the one this child + // runs itself, long before the schedule's first (unref'ed) tick. }); const runtime = new rt.Runtime({ cluster: false }); const kernel = runtime.getKernel(); diff --git a/packages/cli/src/commands/migrate/audit-mail-bodies.ts b/packages/cli/src/commands/migrate/audit-mail-bodies.ts index ccdfa17fe38..b6e1355a74a 100644 --- a/packages/cli/src/commands/migrate/audit-mail-bodies.ts +++ b/packages/cli/src/commands/migrate/audit-mail-bodies.ts @@ -22,6 +22,7 @@ import { OCCUPANCY_HINT, probeMigrationTarget } from '../../utils/migrate-occupa import { describeOccupancy, JOURNAL_MODE_SWITCH_NOTE } from '../../utils/sqlite-occupancy.js'; import { buildDataMigrationPlugins } from '../../utils/data-migration-plugins.js'; import { absentTableReads } from '../../utils/absent-table-reads.js'; +import { exitOneShotCommand } from '../../utils/one-shot-exit.js'; /** * [#24899] What the command did about one declared archive target: `examined` @@ -145,7 +146,19 @@ export default class MigrateAuditMailBodies extends Command { json: Flags.boolean({ description: 'Output as JSON (implies non-interactive; requires --yes to apply)' }), }; + /** + * [#24899] The process must end when the run does — the reason `os migrate + * plan`'s does (#13027): the boot connects the declared datasources + * (`composeDeclaredDatasources`), and a pool it opened can leave the event + * loop alive after the kernel reports a clean shutdown. The failure paths + * exit through oclif's own signal and do not come back here. + */ async run(): Promise { + await this.mask(); + await exitOneShotCommand(typeof process.exitCode === 'number' ? process.exitCode : 0); + } + + private async mask(): Promise { const { flags } = await this.parse(MigrateAuditMailBodies); const timer = createTimer(); const apply = flags.apply; @@ -211,6 +224,10 @@ export default class MigrateAuditMailBodies extends Command { jsonOutput: flags.json, databaseUrl: flags['database-url'], extraPlugins: await buildDataMigrationPlugins({ audit: true }), + // [#24899] Connect the declared datasources as `os serve` does, so a + // declared archive datasource is registered under its name and the + // archive run below reaches it. + composeDeclaredDatasources: true, ...(apply ? {} : { deferSchemaDdl: true, readOnlyProbe: true }), }); } catch (error: any) { From 8a87750ccd3c34cf49c3b415c471714e09865c99 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 15:36:15 +0000 Subject: [PATCH 05/12] feat(cli,plugin-audit): os migrate audit-mail-bodies masks the archive datasource too The Archiver copies sys_audit_log rows past the hot window to the archive datasource verbatim, so copies archived before the writer masked kept the mail body in clear out of the hot run's reach. The command now runs the same mask through the declared archive target's driver, and refuses to report a runtime archive datasource it cannot connect as clean. Claude-Session: https://claude.ai/code/session_01WQWCrorWxW5zDh4n6jCEF5 Co-authored-by: Claude --- .changeset/24899-archived-mail-bodies.md | 16 ++++++++++ content/docs/deployment/cli.mdx | 37 +++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 .changeset/24899-archived-mail-bodies.md diff --git a/.changeset/24899-archived-mail-bodies.md b/.changeset/24899-archived-mail-bodies.md new file mode 100644 index 00000000000..e7a7bd3e007 --- /dev/null +++ b/.changeset/24899-archived-mail-bodies.md @@ -0,0 +1,16 @@ +--- +'@objectstack/cli': minor +'@objectstack/plugin-audit': minor +--- + +feat(cli,plugin-audit): `os migrate audit-mail-bodies` also masks the mail bodies in the audit copies already moved to the `archive` datasource + +Clause-②: yes (widening) + +**Re-run `os migrate audit-mail-bodies --apply` on a deployment with an archive datasource.** The 18.0.0 command masked the hot audit tables only. + +- **What was left in clear.** `sys_audit_log` declares `lifecycle.archive: { after: '90d', to: 'archive', keep: '7y' }`. Past the hot window, the lifecycle Archiver copies its rows to the datasource named `archive` as they are, then deletes them hot. A `sys_email` / `sys_identity_email` copy archived before the audit writer masked the mail body still holds `body_text` / `body_html` in clear there, password-reset, verification, magic-link and invitation links included, for as long as the archive keeps it. The engine never routes `sys_audit_log` to the archive, so the hot run could not reach those rows. +- **What changed.** After the hot run, the command reads the archive target each copy table declares (`sys_audit_log` only; `sys_activity` declares none) and runs the same mask, in the same dry-run or `--apply` mode, through that datasource's driver. Each archived row keeps every other column, `updated_at` included. The run is idempotent and its report is counts only. A hot copy masked first is archived masked, so one run covers both the copies already archived and the ones still to be. +- **Which archives it reaches.** The command now connects the datasources the stack config declares, as `os serve` does. An archive datasource is declared with `autoConnect: true`, since nothing routes objects to it. If the archive is instead a runtime datasource created in Setup → Datasources, which `os serve` connects and this command does not, the run reports it `not-connected` and exits `1` rather than reporting a clean archive. Declare it in the config's `datasources` and re-run. +- **New in the `--json` document:** an `archive` array, one entry per declared target: `{ datasource, objects, status, report? }`, with `status` one of `examined`, `none`, `not-connected` or `unknown`. The `report` key and its shape are unchanged. +- **New exports from `@objectstack/plugin-audit`:** `migrateArchivedMailLogBodyCopies(driver, logger, options)`, the runner over a datasource's driver, and an `objects` option on both runners, which limits a run to the named copy tables. Types `MailLogBodyArchiveDriver`, `MailLogBodyCopyObject` and `MailLogBodyMigrationOptions`. diff --git a/content/docs/deployment/cli.mdx b/content/docs/deployment/cli.mdx index b888e1dd5a4..43192587dcc 100644 --- a/content/docs/deployment/cli.mdx +++ b/content/docs/deployment/cli.mdx @@ -1204,7 +1204,7 @@ where the data lives. | `os migrate comment-reactions` | Carry the emoji reactions a 17.x deployment stored in the retired `sys_comment.reactions` column into `sys_comment_reaction` records, list every stored value that does not read, and record the verified run that lets `os migrate apply --allow-destructive` drop the column | | `os migrate organization-ownership` | The v18 ownership ceremony (ADR-0131 D10): write a plan of what each table's rows will become, then `--apply` that reviewed plan. It upgrades a database 17.x or an 18.0 pre-release (`18.0.0-next.*`) created; a v18 server refuses to start on one until it has run. See [below](#os-migrate-organization-ownership) | | `os migrate activity-changed-fields` | Stamp the two read-rule columns (`changed_fields`, `milestone_key`) on the `sys_activity` rows written before they existed. Until it runs, those rows are withheld from every reader but the system, so run it at once after upgrading. No flag; idempotent | -| `os migrate audit-mail-bodies` | Mask the mail bodies (`body_text`, `body_html`) that `sys_audit_log` and `sys_activity` copies of `sys_email` / `sys_identity_email` rows still hold in clear, written before the audit writer masked them. Reports counts only. No flag; idempotent. See [below](#os-migrate-audit-mail-bodies) | +| `os migrate audit-mail-bodies` | Mask the mail bodies (`body_text`, `body_html`) that `sys_audit_log` and `sys_activity` copies of `sys_email` / `sys_identity_email` rows still hold in clear, written before the audit writer masked them. Covers the `archive` datasource `sys_audit_log` archives to as well. Reports counts only. No flag; idempotent. See [below](#os-migrate-audit-mail-bodies) | | `os migrate comment-parent-pair` | Stamp the parent pair (`object_name`, `record_id`) on the `sys_comment` rows written before it existed. Until it runs, a broad comment read past 2,000 candidate rows omits comments the reader may see, and its total agrees with the short list, so run it at once after upgrading. No flag; idempotent | | `os migrate meta --stored` | Replay the metadata conversion chain over this deployment's `sys_metadata` rows and rewrite the ones still carrying a pre-protocol shape. Hygiene, not a gate — nothing depends on it having run | | `os migrate security-catalog-overlays` | List the environment-wide permission-set and position rows (the legacy plurals `permissions` / `positions` included) stored over a name a configured package holds — the rows a v18 cold boot refuses — each with the package that holds its name; `--apply` deletes them, one history tombstone per row. Run it before the first v18 boot, with the flags and environment the deployment boots with: the platform security plugin's sets are held only where `os serve` composes it (an auth secret set, and the `auth` tier on), and the step takes `serve`'s `--preset` and `--dev` with `serve`'s meaning. Adopts nothing; exits 1 while it lists a row | @@ -1505,6 +1505,41 @@ re-running is how an interrupted run is finished. A failed read or write exits for the audit log, unless the deployment lengthens it); run it once after upgrading rather than wait for that. +**The archive datasource too.** `sys_audit_log` declares +`lifecycle.archive: { after: '90d', to: 'archive', keep: '7y' }`: past the hot +window, the lifecycle Archiver copies its rows to the datasource named +`archive` as they are, and deletes them hot. A copy archived before the writer +masked still holds the body there, for up to seven years. After the hot run, +the command reads the archive target `sys_audit_log` declares and runs the same +mask, in the same dry-run or `--apply` mode, over that datasource. Its rows keep +every other column, `updated_at` included. A hot copy masked first is archived +masked, so one run covers both the copies already archived and the ones still +to be. `sys_activity` declares no archive: its copies are only ever hot. + +The command connects the datasources the stack config declares, as `os serve` +does, so it reaches an archive declared there. A managed datasource nothing +routes to is connected only with `autoConnect: true`, which is how an archive +datasource is declared: + +```ts +datasources: [ + { name: 'archive', driver: 'sqlite', config: { filename: './data/archive.db' }, autoConnect: true }, +], +``` + +The `--json` document carries one `archive` entry per declared target, with a +`status`: + +| `status` | Meaning | Exit | +| :--- | :--- | :--- | +| `examined` | Connected and masked (or counted); `report` holds the counts | `0`, or `1` on a failed read or write | +| `none` | No such datasource, and no runtime datasource row of that name: the Archiver retains every row hot, so nothing was archived | `0` | +| `not-connected` | The database holds an active runtime datasource of that name (one created in Setup → Datasources). `os serve` connects it; this command does not, so the archive was **not** examined | `1` | +| `unknown` | Not connected, and the datasource rows could not be read | `1` | + +On `not-connected`, declare the archive datasource in the stack config's +`datasources` (with `autoConnect: true`), then re-run. + #### `os migrate organization-ownership` The v18 ownership ceremony (ADR-0131 D10), and the upgrade path of a database From 39fc914f768bae4e1f2bc09eeb3d08e3fd8c82fb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 16:29:31 +0000 Subject: [PATCH 06/12] test(cli): declare the archive mask pin's measured budget Claude-Session: https://claude.ai/code/session_01WQWCrorWxW5zDh4n6jCEF5 Co-authored-by: Claude --- .../migrate/audit-mail-bodies.archive.integration.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts b/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts index ebad5bc68e5..399088d6223 100644 --- a/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts +++ b/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts @@ -1,4 +1,5 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// test-budget: 70 — spawns the real producer once per single-cause fixture: two served-shape boots that run the real Archiver, and five spawns of the source CLI (dry run, --apply, re-run over the archive fixture; --apply on a project with no archive; a dry run with a runtime archive row); 54.7 s, the slower of two local runs on a shared container, x 1.25, no CI reading yet /** * [#24899] `os migrate audit-mail-bodies` masks the mail-log bodies the From 3340a4ad4b9a46e5ce72ed4b8cf92bc63933dccb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 16:39:17 +0000 Subject: [PATCH 07/12] docs(audits): regenerate the tenant-audit census for the archive mask's write site Generator output only (node scripts/tenant-audit-census.mjs --write). Claude-Session: https://claude.ai/code/session_01WQWCrorWxW5zDh4n6jCEF5 Co-authored-by: Claude --- .../docs/permissions/tenant-audit-census.mdx | 20 +++++++++---------- ...08-tenant-audit-write-call-sites.counts.md | 17 ++++++++-------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/content/docs/permissions/tenant-audit-census.mdx b/content/docs/permissions/tenant-audit-census.mdx index 9bdb3e093c8..ac502afbfec 100644 --- a/content/docs/permissions/tenant-audit-census.mdx +++ b/content/docs/permissions/tenant-audit-census.mdx @@ -223,23 +223,23 @@ cannot read, and they are neither in nor out. | what | count | | :--- | ---: | -| write call sites on the application surface | **232** | +| write call sites on the application surface | **233** | | …whose object name is statically decidable | 156 | -| …whose object name is chosen at run time | 76 | +| …whose object name is chosen at run time | 77 | | …against an object with tenancy ENABLED | 155 | | …against an object that declares tenancy off | 1 | | threading a tenant context | 174 | | PROVABLY carrying none (options read, no context key) | **8** | | …of those, against a decidably tenancy-enabled object | **2** | -| options argument UNREADABLE — may or may not carry one | 50 | +| options argument UNREADABLE — may or may not carry one | 51 | | …of those, against a decidably tenancy-enabled object | 29 | | threading a decidably ELEVATED (`isSystem`) context | 131 | | threading a context that is decidably NOT elevated | 0 | -| threading a context whose elevation is a run-time fact | 93 | +| threading a context whose elevation is a run-time fact | 94 | | how the instrument reached the site | count | | :--- | ---: | -| receiver carried a readable engine type | 191 | +| receiver carried a readable engine type | 192 | | receiver erased, placed by the object NAME | 31 | | receiver erased, placed by an `object: string` PARAMETER | 5 | | receiver erased, placed by an `UNTYPED_RECEIVERS` row | 5 | @@ -247,7 +247,7 @@ cannot read, and they are neither in nor out. | object name spelled inline | 107 | | object name spelled through a `const` | 49 | | object name is an `object: string` parameter | 7 | -| object name is some other run-time expression | 69 | +| object name is some other run-time expression | 70 | ### Subtractions the census could NOT defend — enforced @@ -297,13 +297,13 @@ holds still. They are required to be HERE and to say WHEN they were true; their values are not compared. The reasoning, and the measurement behind it, are in `scripts/check-tenant-audit-census.mjs`. -Measured on 2026-10-20 at `300e9d436`. +Measured on 2026-10-20 at `39fc914f7`. | corpus scale (not enforced) | count | | :--- | ---: | -| tracked non-test sources scanned | 675 | -| engine-shaped types recognised | 80 | +| tracked non-test sources scanned | 681 | +| engine-shaped types recognised | 82 | | declared objects in the registry | 118 | -| same-named calls subtracted as non-engine | 166 | +| same-named calls subtracted as non-engine | 169 | {/* END GENERATED: tenant-audit-census */} diff --git a/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md b/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md index 61122fb75b6..51bf6dc291e 100644 --- a/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md +++ b/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md @@ -33,19 +33,19 @@ silent, and `node scripts/tenant-audit-census.mjs --write` is the resolution. | Measure | Value | |---|---:| -| Write call sites | 232 | +| Write call sites | 233 | | Object name statically decidable | 156 | -| Object name chosen at run time | 76 | +| Object name chosen at run time | 77 | | Against a tenancy-enabled object | 155 | | Against an object declaring tenancy off | 1 | | Threading a tenant context | 174 | | Provably carrying none | 8 | | …and decidably tenancy-enabled | 2 | -| Options argument unreadable | 50 | +| Options argument unreadable | 51 | | …and decidably tenancy-enabled | 29 | | Threading a decidably elevated context | 131 | | Threading a decidably non-elevated context | 0 | -| Threading a context of undecidable elevation | 93 | +| Threading a context of undecidable elevation | 94 | ## Subtractions the census could NOT defend — enforced @@ -90,14 +90,14 @@ holds still. They are required to be HERE and to say WHEN they were true; their values are not compared. The reasoning, and the measurement behind it, are in `scripts/check-tenant-audit-census.mjs`. -Measured on 2026-10-20 at `300e9d436`. +Measured on 2026-10-20 at `39fc914f7`. | corpus scale (not enforced) | count | | :--- | ---: | -| tracked non-test sources scanned | 675 | -| engine-shaped types recognised | 80 | +| tracked non-test sources scanned | 681 | +| engine-shaped types recognised | 82 | | declared objects in the registry | 118 | -| same-named calls subtracted as non-engine | 166 | +| same-named calls subtracted as non-engine | 169 | ## Every site @@ -121,6 +121,7 @@ Measured on 2026-10-20 at `300e9d436`. | `packages/plugins/plugin-audit/src/auth-event-audit.ts` | `insert` | `sys_audit_log` | enabled | options unreadable | 1 | | `packages/plugins/plugin-audit/src/comment-parent-pair-backfill.ts` | `update` | `sys_comment` | enabled | elevated | 1 | | `packages/plugins/plugin-audit/src/mail-log-body-migration.ts` | `update` | `object` | undecidable | elevated | 1 | +| `packages/plugins/plugin-audit/src/mail-log-body-migration.ts` | `update` | `object` | undecidable | options unreadable | 1 | | `packages/plugins/plugin-audit/src/read-audit.ts` | `insert` | `sys_audit_log` | enabled | options unreadable | 1 | | `packages/plugins/plugin-audit/src/stored-metadata-body-migration.ts` | `update` | `object` | undecidable | elevated | 1 | | `packages/plugins/plugin-auth/src/admin-import-users.ts` | `insert` | `sys_audit_log` | enabled | options unreadable | 1 | From 105a8110325fce2da732102678a206e5eec38763 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 16:42:35 +0000 Subject: [PATCH 08/12] docs(audits,cli): hold the census prose and the composeDeclaredDatasources caller list to this diff The ten hand-written census figures this diff moved, each +1, and the caller list that os migrate audit-mail-bodies now joins. Claude-Session: https://claude.ai/code/session_01WQWCrorWxW5zDh4n6jCEF5 Co-authored-by: Claude --- content/docs/permissions/tenant-audit-census.mdx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/content/docs/permissions/tenant-audit-census.mdx b/content/docs/permissions/tenant-audit-census.mdx index ac502afbfec..432d2d7ab25 100644 --- a/content/docs/permissions/tenant-audit-census.mdx +++ b/content/docs/permissions/tenant-audit-census.mdx @@ -122,7 +122,7 @@ are reported as `undecidable` rather than assumed either way. The same holds twice over for the context. An options argument spelled as a literal can be read; one spelled `options`, `{ ...opts }`, or handed through a -forwarding shim cannot, and **50 of the 232 sites are spelled that way**. A +forwarding shim cannot, and **51 of the 233 sites are spelled that way**. A context resolved from an inline literal or a local `const` can be tested for `isSystem`; one arriving from a helper call cannot. @@ -150,8 +150,8 @@ now **0**: nothing on this surface threads a context that provably lacks the fla **"No tenant context" counted sites it had not read.** An options argument the walker could not parse was folded into the same bucket as one it had read and -found empty. That published **58 sites "carrying no tenant context at all"** -when 8 said so and 50 were simply unread — an over-claim in the *alarming* +found empty. That published **59 sites "carrying no tenant context at all"** +when 8 said so and 51 were simply unread — an over-claim in the *alarming* direction, on the very figure this page tells other cards to cite. `carries` is now three-valued, and an unreadable argument can never contribute to the provable count. @@ -187,10 +187,10 @@ reproduce them. Where it disagrees, it disagrees on the page: | carried figure | where it survives | this census | | :--- | :--- | ---: | -| 175 write call sites | quoted in the merged changeset | **232** | +| 175 write call sites | quoted in the merged changeset | **233** | | 24 carrying no tenant context | quoted in the merged changeset | **2** provable and tenancy-enabled; **29** more whose options argument is unreadable | -| 127 of 175 statically decidable, 48 runtime-parameter-name sites | restated on the `isSystem`-scoping card | **156 of 232** decidable, **76** undecidable | -| 135 (77%) silenced by the `isSystem` guard before the posture gate | the lost issue body — **no surviving corroboration** | **not reproduced**: 131 decidably elevated, 0 decidably not, 93 undecidable | +| 127 of 175 statically decidable, 48 runtime-parameter-name sites | restated on the `isSystem`-scoping card | **156 of 233** decidable, **77** undecidable | +| 135 (77%) silenced by the `isSystem` guard before the posture gate | the lost issue body — **no surviving corroboration** | **not reproduced**: 131 decidably elevated, 0 decidably not, 94 undecidable | | 141 and 132, two independent re-derivations | the card that filed this work | — | **The differences are not reconciled, and deliberately so.** The old census's @@ -207,11 +207,11 @@ would report a smaller number and would not say so. The fourth row is the one worth flagging to anyone citing it. **The 135 / 77% figure has no surviving corroboration anywhere in the tree.** This census reads -131 of 232 (56%) as decidably elevated, with 93 more whose elevation is a +131 of 233 (56%) as decidably elevated, with 93 more whose elevation is a run-time fact — so the claim is neither confirmed nor refuted, and the honest answer is that a static reading cannot settle it. -⇒ **Cite `2 / 232`, and say what it is**: the sites whose options argument was +⇒ **Cite `2 / 233`, and say what it is**: the sites whose options argument was READ and holds no tenant context, against a decidably tenancy-enabled object. That is the control's provable yield surface. ⛔ Do not cite it as "the sites without tenant context" — **29 further sites** have an options argument this From f72911d847f0170ae1456f70f388e73fc1b16d1b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 16:42:57 +0000 Subject: [PATCH 09/12] docs(cli): os migrate audit-mail-bodies joins composeDeclaredDatasources' caller list The previous commit's message names this change; it landed here. Claude-Session: https://claude.ai/code/session_01WQWCrorWxW5zDh4n6jCEF5 Co-authored-by: Claude --- packages/cli/src/utils/schema-migrate.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/utils/schema-migrate.ts b/packages/cli/src/utils/schema-migrate.ts index f97922f7ea0..27841c8de94 100644 --- a/packages/cli/src/utils/schema-migrate.ts +++ b/packages/cli/src/utils/schema-migrate.ts @@ -1174,8 +1174,10 @@ export async function bootSchemaStack( * — and plan each one the deployment's objects live in beside the primary * ({@link SchemaStack.drivers}). Set by `os migrate plan` and `os migrate * apply`, so the plan an operator reads and the apply that confirms it - * cover the same datasources, and by `os migrate unmapped-columns`, a - * filter over that plan's own findings; by nothing else. + * cover the same datasources, by `os migrate unmapped-columns`, a + * filter over that plan's own findings, and by `os migrate + * audit-mail-bodies`, which masks the archive datasource a declaration + * names (#24899); by nothing else. * * Without it a `datasourceMapping` rule routing objects to a declared * datasource left them bound to no driver here, the declared DEFAULT's From 4fb9afa11c69c5542591f1cc16f93725c24cf85e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 16:45:36 +0000 Subject: [PATCH 10/12] docs(audits): the census page's unchecked elevation figure, 93 to 94 Claude-Session: https://claude.ai/code/session_01WQWCrorWxW5zDh4n6jCEF5 Co-authored-by: Claude --- content/docs/permissions/tenant-audit-census.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/permissions/tenant-audit-census.mdx b/content/docs/permissions/tenant-audit-census.mdx index 432d2d7ab25..afe0f643350 100644 --- a/content/docs/permissions/tenant-audit-census.mdx +++ b/content/docs/permissions/tenant-audit-census.mdx @@ -207,7 +207,7 @@ would report a smaller number and would not say so. The fourth row is the one worth flagging to anyone citing it. **The 135 / 77% figure has no surviving corroboration anywhere in the tree.** This census reads -131 of 233 (56%) as decidably elevated, with 93 more whose elevation is a +131 of 233 (56%) as decidably elevated, with 94 more whose elevation is a run-time fact — so the claim is neither confirmed nor refuted, and the honest answer is that a static reading cannot settle it. From 0f17ba70f50d09725b4fb375e7702cd7808804eb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 17:08:57 +0000 Subject: [PATCH 11/12] wip(cli): a declared archive this boot does not connect reads not-connected Claude-Session: https://claude.ai/code/session_01WQWCrorWxW5zDh4n6jCEF5 Co-authored-by: Claude --- .changeset/24899-archived-mail-bodies.md | 7 +- content/docs/deployment/cli.mdx | 31 ++++--- ...it-mail-bodies.archive.integration.test.ts | 64 +++++++++++--- .../src/commands/migrate/audit-mail-bodies.ts | 83 +++++++++++++++---- 4 files changed, 146 insertions(+), 39 deletions(-) diff --git a/.changeset/24899-archived-mail-bodies.md b/.changeset/24899-archived-mail-bodies.md index e7a7bd3e007..e5925aa6f8d 100644 --- a/.changeset/24899-archived-mail-bodies.md +++ b/.changeset/24899-archived-mail-bodies.md @@ -11,6 +11,9 @@ Clause-②: yes (widening) - **What was left in clear.** `sys_audit_log` declares `lifecycle.archive: { after: '90d', to: 'archive', keep: '7y' }`. Past the hot window, the lifecycle Archiver copies its rows to the datasource named `archive` as they are, then deletes them hot. A `sys_email` / `sys_identity_email` copy archived before the audit writer masked the mail body still holds `body_text` / `body_html` in clear there, password-reset, verification, magic-link and invitation links included, for as long as the archive keeps it. The engine never routes `sys_audit_log` to the archive, so the hot run could not reach those rows. - **What changed.** After the hot run, the command reads the archive target each copy table declares (`sys_audit_log` only; `sys_activity` declares none) and runs the same mask, in the same dry-run or `--apply` mode, through that datasource's driver. Each archived row keeps every other column, `updated_at` included. The run is idempotent and its report is counts only. A hot copy masked first is archived masked, so one run covers both the copies already archived and the ones still to be. -- **Which archives it reaches.** The command now connects the datasources the stack config declares, as `os serve` does. An archive datasource is declared with `autoConnect: true`, since nothing routes objects to it. If the archive is instead a runtime datasource created in Setup → Datasources, which `os serve` connects and this command does not, the run reports it `not-connected` and exits `1` rather than reporting a clean archive. Declare it in the config's `datasources` and re-run. -- **New in the `--json` document:** an `archive` array, one entry per declared target: `{ datasource, objects, status, report? }`, with `status` one of `examined`, `none`, `not-connected` or `unknown`. The `report` key and its shape are unchanged. +- **Which archives it reaches.** The command now connects the managed datasources the stack config declares, through the same connect service `os serve` uses. An archive datasource is declared with `autoConnect: true`, since nothing routes objects to it. It is NOT examined, and the run reports `not-connected` and exits `1` rather than reporting a clean archive, in two cases. In the first, the archive is declared but this boot did not connect it; the entry carries the connect status and reason. In the second, the archive is a runtime datasource created in Setup → Datasources, which `os serve` connects and this command does not; declare it in the config's `datasources` and re-run. +- **What `none` means.** No datasource of that name is declared and connected on this boot, none was attempted, and no active runtime datasource row names one. That is all it checks. An archive a host registers in code outside the stack config is not reached and reads `none`. To mask it, declare it in the stack config's `datasources` with `autoConnect: true`. +- **The command's boot can now fail on a declared datasource.** A bound, mapped or boot-critical declared datasource that cannot connect now fails the command's boot with exit `1`, as it fails `os serve`'s. `OS_ALLOW_DRIVER_CONNECT_FAILURE` is the same escape. +- **Run it with the server stopped**, or re-run it once and read 0 to mask in both halves. A live server's Archiver can move a row between the hot run and the archive run. +- **New in the `--json` document:** an `archive` array, one entry per declared target: `{ datasource, objects, status, report?, connection? }`, with `status` one of `examined`, `none`, `not-connected` or `unknown`. The `report` key and its shape are unchanged. - **New exports from `@objectstack/plugin-audit`:** `migrateArchivedMailLogBodyCopies(driver, logger, options)`, the runner over a datasource's driver, and an `objects` option on both runners, which limits a run to the named copy tables. Types `MailLogBodyArchiveDriver`, `MailLogBodyCopyObject` and `MailLogBodyMigrationOptions`. diff --git a/content/docs/deployment/cli.mdx b/content/docs/deployment/cli.mdx index 43192587dcc..b7d8d69e440 100644 --- a/content/docs/deployment/cli.mdx +++ b/content/docs/deployment/cli.mdx @@ -1501,9 +1501,15 @@ value. **Idempotent**: a masked body is the mask, so a second run masks nothing, and re-running is how an interrupted run is finished. A failed read or write exits -`1`. The copies age out with each table's retention (14 days for activity, 90 -for the audit log, unless the deployment lengthens it); run it once after -upgrading rather than wait for that. +`1`. Activity copies age out with its 14-day retention. Audit-log copies do not +age out: with no archive datasource they are retained hot, and with one they +are moved there after 90 days and kept for seven years. Run it once after +upgrading. + +**Run it with the server stopped**, or re-run it once and read 0 to mask in +both halves. A live server's Archiver sweeps on its own schedule, and a row it +moves between the hot run and the archive run can land in the archive +unmasked. **The archive datasource too.** `sys_audit_log` declares `lifecycle.archive: { after: '90d', to: 'archive', keep: '7y' }`: past the hot @@ -1516,9 +1522,9 @@ every other column, `updated_at` included. A hot copy masked first is archived masked, so one run covers both the copies already archived and the ones still to be. `sys_activity` declares no archive: its copies are only ever hot. -The command connects the datasources the stack config declares, as `os serve` -does, so it reaches an archive declared there. A managed datasource nothing -routes to is connected only with `autoConnect: true`, which is how an archive +The command connects the managed datasources the stack config declares through +the same connect service `os serve` uses. A managed datasource nothing routes +to is connected only with `autoConnect: true`, which is how an archive datasource is declared: ```ts @@ -1533,12 +1539,17 @@ The `--json` document carries one `archive` entry per declared target, with a | `status` | Meaning | Exit | | :--- | :--- | :--- | | `examined` | Connected and masked (or counted); `report` holds the counts | `0`, or `1` on a failed read or write | -| `none` | No such datasource, and no runtime datasource row of that name: the Archiver retains every row hot, so nothing was archived | `0` | -| `not-connected` | The database holds an active runtime datasource of that name (one created in Setup → Datasources). `os serve` connects it; this command does not, so the archive was **not** examined | `1` | +| `none` | No datasource of that name is declared and connected on this boot, none was attempted, and no active runtime datasource row names one. That is all it checked | `0` | +| `not-connected` | The archive was **not** examined. Either the stack config declares it and this boot did not connect it (`connection` carries the connect status and reason), or the database holds an active runtime datasource of that name (one created in Setup → Datasources), which `os serve` connects and this command does not | `1` | | `unknown` | Not connected, and the datasource rows could not be read | `1` | -On `not-connected`, declare the archive datasource in the stack config's -`datasources` (with `autoConnect: true`), then re-run. +On `not-connected` for a runtime datasource, declare the archive datasource in +the stack config's `datasources` (with `autoConnect: true`), then re-run. For a +declared one, make it reachable from where the command runs, then re-run. + +An archive that a host registers in code outside the stack config (a plugin +that publishes the driver itself) is not reached and reads `none`. To mask it, +declare it in the stack config's `datasources` with `autoConnect: true`. #### `os migrate organization-ownership` diff --git a/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts b/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts index 399088d6223..5ae8434ce67 100644 --- a/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts +++ b/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts @@ -1,5 +1,5 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. -// test-budget: 70 — spawns the real producer once per single-cause fixture: two served-shape boots that run the real Archiver, and five spawns of the source CLI (dry run, --apply, re-run over the archive fixture; --apply on a project with no archive; a dry run with a runtime archive row); 54.7 s, the slower of two local runs on a shared container, x 1.25, no CI reading yet +// test-budget: 70 — spawns the real producer once per single-cause fixture: two served-shape boots that run the real Archiver, and six spawns of the source CLI (dry run, --apply, re-run over the archive fixture; --apply on a project with no archive; a dry run with a runtime archive row; --apply with a declared archive that cannot connect); 54.7 s, the slower of two local runs on a shared container, x 1.25, no CI reading yet /** * [#24899] `os migrate audit-mail-bodies` masks the mail-log bodies the @@ -40,6 +40,10 @@ * connects and this command's boot does not) is NOT reported clean: the * archive reads `not-connected` and the command exits 1. With neither a * connected datasource nor such a row it reads `none` and exits 0. + * 7. A DECLARED `autoConnect: true` archive this boot cannot connect is NOT + * reported clean either: gate (c) is not fail-fast, so the boot degrades + * and leaves no driver, and the archive reads `not-connected`, carrying + * the connect service's verdict, and the command exits 1. * * ⛔ The bodies are sentinels, never a link or a token, and every comparison is * reported as a count or a boolean. @@ -51,6 +55,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { isDeepStrictEqual } from 'node:util'; import Database from 'better-sqlite3'; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -167,6 +172,11 @@ let hotAfterApply: Record[]; let archivedAfterAging: Record[]; let runtimeRowRun: Run; let noArchiveRun: Run; +let unreachableDir: string; +let unreachableRun: Run; +let archivedAfterDry: Record[]; +let hotBeforeDry: Record[]; +let hotAfterDry: Record[]; function sweep(mode: 'seed' | 'age'): { source: string; archived: number | null } { const r = spawnSync(process.execPath, ['--input-type=module', '-e', SWEEP_CHILD], { @@ -243,7 +253,10 @@ beforeAll(async () => { seedDetail = sweep('seed'); archivedBefore = auditRows(archiveDb); + hotBeforeDry = auditRows(mainDb); runs.dry = await runCommand(dir, mainDb, ['--json']); + archivedAfterDry = auditRows(archiveDb); + hotAfterDry = auditRows(mainDb); runs.apply = await runCommand(dir, mainDb, ['--apply', '--yes', '--json']); archivedAfterApply = auditRows(archiveDb); hotAfterApply = auditRows(mainDb); @@ -267,14 +280,27 @@ beforeAll(async () => { db.close(); } runtimeRowRun = await runCommand(runtimeRowDir, runtimeRowDb, ['--json']); + + // A declared archive whose SQLite file cannot be opened: its directory is a + // regular file, so the connect fails on every attempt. + unreachableDir = mkdtempSync(join(tmpdir(), 'os-24899-unreachable-')); + writeFileSync(join(unreachableDir, 'blocker'), 'not a directory'); + writeArtifact(unreachableDir, [ + { name: 'archive', driver: 'sqlite', config: { filename: join(unreachableDir, 'blocker', 'archive.db') }, autoConnect: true }, + ]); + unreachableRun = await runCommand(unreachableDir, join(unreachableDir, 'main.db'), ['--apply', '--yes', '--json']); }, HOOK_TIMEOUT_MS); afterAll(() => { if (dir) rmSync(dir, { recursive: true, force: true }); if (runtimeRowDir) rmSync(runtimeRowDir, { recursive: true, force: true }); + if (unreachableDir) rmSync(unreachableDir, { recursive: true, force: true }); }); const doc = (run: Run): any => JSON.parse(run.stdout); +/** Two row sets hold the same rows, column for column — a boolean, so a failure prints no row. */ +const sameRows = (a: Record[], b: Record[]): boolean => + a.length === b.length && a.every((row, i) => isDeepStrictEqual(row, b[i])); const archiveOf = (run: Run): any => doc(run).archive.find((a: { datasource: string }) => a.datasource === 'archive'); describe('[#24899] os migrate audit-mail-bodies over an archive datasource', () => { @@ -293,8 +319,10 @@ describe('[#24899] os migrate audit-mail-bodies over an archive datasource', () objects: ['sys_audit_log'], report: { apply: false, failures: 0, byObject: { sys_audit_log: { scanned: 2, rewritten: 2 } } }, }); - // Written only by --apply below: the dry run left the archive as the Archiver did. - expect(clear(archivedBefore)).toBe(2); + // Both stores, read after the dry run: neither moved. + expect(sameRows(archivedAfterDry, archivedBefore)).toBe(true); + expect(sameRows(hotAfterDry, hotBeforeDry)).toBe(true); + expect(clear(archivedAfterDry)).toBe(2); }); it('2. --apply leaves 0 clear copies in either store, and every other archived row is untouched', () => { @@ -304,14 +332,15 @@ describe('[#24899] os migrate audit-mail-bodies over an archive datasource', () expect(clear(archivedAfterApply)).toBe(0); expect(clear(hotAfterApply)).toBe(0); const before = new Map(archivedBefore.map((r) => [r.id, r])); - for (const row of archivedAfterApply) { - const original = before.get(row.id)!; - for (const key of Object.keys(original)) { - if (key === 'new_value' || key === 'old_value') continue; - expect(row[key], `${String(row.id)}.${key}`).toEqual(original[key]); - } - } - expect(archivedAfterApply.find((r) => r.id === 'aud_old_3')).toEqual(before.get('aud_old_3')); + const movedColumns = archivedAfterApply.flatMap((row) => + Object.keys(before.get(row.id)!) + .filter((key) => key !== 'new_value' && key !== 'old_value') + .filter((key) => !isDeepStrictEqual(row[key], before.get(row.id)![key])) + .map((key) => `${String(row.id)}.${key}`), + ); + // Names only: an id and a column, never a value. + expect(movedColumns).toEqual([]); + expect(isDeepStrictEqual(archivedAfterApply.find((r) => r.id === 'aud_old_3'), before.get('aud_old_3'))).toBe(true); const masked = JSON.parse(String(archivedAfterApply.find((r) => r.id === 'aud_old_1')!.new_value)); expect([masked.body_text === MASK, masked.body_html === MASK, masked.subject === 'Subject kept']).toEqual([true, true, true]); }); @@ -321,7 +350,7 @@ describe('[#24899] os migrate audit-mail-bodies over an archive datasource', () expect(run.code, run.stderr.slice(-2000)).toBe(0); expect(doc(run).report.rewritten).toBe(0); expect(archiveOf(run).report).toMatchObject({ scanned: 2, rewritten: 0, failures: 0 }); - expect(archivedAfterAgain).toEqual(archivedAfterApply); + expect(sameRows(archivedAfterAgain, archivedAfterApply)).toBe(true); }); it('4. the copy that was hot when the command ran is archived masked', () => { @@ -332,7 +361,7 @@ describe('[#24899] os migrate audit-mail-bodies over an archive datasource', () }); it('5. no run prints a body', () => { - for (const run of [runs.dry, runs.apply, runs.again, runtimeRowRun, noArchiveRun]) { + for (const run of [runs.dry, runs.apply, runs.again, runtimeRowRun, noArchiveRun, unreachableRun]) { expect(`${run.stdout}\n${run.stderr}`.includes(SENTINEL)).toBe(false); } }); @@ -343,4 +372,13 @@ describe('[#24899] os migrate audit-mail-bodies over an archive datasource', () expect(runtimeRowRun.code).toBe(1); expect(archiveOf(runtimeRowRun)).toEqual({ datasource: 'archive', objects: ['sys_audit_log'], status: 'not-connected' }); }); + + it('7. a declared archive this boot cannot connect is NOT reported clean', () => { + expect(unreachableRun.code).toBe(1); + const entry = archiveOf(unreachableRun); + expect(entry).toMatchObject({ datasource: 'archive', objects: ['sys_audit_log'], status: 'not-connected' }); + expect(typeof entry.connection?.status).toBe('string'); + expect(entry.connection.status === 'connected' || entry.connection.status === 'already-registered').toBe(false); + expect(entry.report).toBeUndefined(); + }); }); diff --git a/packages/cli/src/commands/migrate/audit-mail-bodies.ts b/packages/cli/src/commands/migrate/audit-mail-bodies.ts index b6e1355a74a..7916b2bc131 100644 --- a/packages/cli/src/commands/migrate/audit-mail-bodies.ts +++ b/packages/cli/src/commands/migrate/audit-mail-bodies.ts @@ -25,18 +25,54 @@ import { absentTableReads } from '../../utils/absent-table-reads.js'; import { exitOneShotCommand } from '../../utils/one-shot-exit.js'; /** - * [#24899] What the command did about one declared archive target: `examined` - * (its driver is registered on this boot, and `report` is the run), `none` (no - * connected datasource and no runtime row of that name — the deployment has no - * archive), `not-connected` (an active runtime datasource row of that name - * exists, which this boot does not connect: NOT examined) or `unknown` (not - * connected, and that row could not be read). The last two fail the run. + * [#24899] What the command did about one declared archive target: + * + * - `examined`: its driver is registered on this boot, and `report` is the run; + * - `not-connected`, NOT examined: either the stack config declares a + * datasource of that name that this boot attempted and did not connect + * (`connection` carries the connect service's retained verdict), or an + * active runtime datasource row of that name exists, which `os serve` + * connects and this boot does not; + * - `unknown`, NOT examined: neither of the above, and the runtime rows could + * not be read; + * - `none`: no datasource of that name is declared and connected on this boot, + * none was attempted, and no active runtime row names one. That is all it + * checked: an archive a host registers in code outside the stack config is + * not visible here. + * + * `not-connected` and `unknown` fail the run. */ interface ArchiveOutcome { datasource: string; objects: string[]; status: 'examined' | 'none' | 'not-connected' | 'unknown'; report?: MailLogBodyMigrationReport; + connection?: { status: string; reason?: string }; +} + +/** + * The connect service's retained verdict for a declared datasource named + * `datasource` that this boot attempted and did not connect, or `undefined` + * when it connected or was never attempted. Gate (c) `autoConnect` is not a + * fail-fast trigger, so a declared archive that fails to connect only degrades + * and leaves no driver behind. The verdicts are the ones + * `findDeclaredDatasourceDrivers` reads. + */ +function declaredConnectFailure(kernel: unknown, datasource: string): { status: string; reason?: string } | undefined { + let states: unknown; + try { + states = (kernel as { getService?: (n: string) => any })?.getService?.('datasource-connection')?.listConnectionStates?.(); + } catch { + return undefined; + } + const state = (Array.isArray(states) ? states : []).find((s: { name?: unknown }) => s?.name === datasource) as + | { status?: unknown; reason?: unknown; availability?: unknown } + | undefined; + if (!state || state.availability === 'available') return undefined; + return { + status: String(state.status), + ...(typeof state.reason === 'string' && state.reason !== '' ? { reason: state.reason } : {}), + }; } /** @@ -110,12 +146,14 @@ async function confirm(question: string): Promise { * moves it is archived masked, so the two runs cover the copies already * archived and the ones still to be. * - * Absence is loud: when the target is not connected here but the database + * Absence is loud: when the target is not connected here but the stack config + * declares it and this boot's connect failed or was refused, or the database * holds an active runtime datasource row of that name (one an admin created, * which `os serve` connects and this boot does not), the archive was NOT - * examined — the report says so and the command exits non-zero. No such row - * and no connected datasource means the deployment has no archive, and the - * Archiver retains every row hot: nothing to mask there. + * examined — the report says so and the command exits non-zero. `none` says + * only what was checked: no such declaration attempted, no such row. An + * archive a host registers in code outside the stack config is not reached; + * declare it in the stack config's `datasources` with `autoConnect: true`. */ export default class MigrateAuditMailBodies extends Command { static override description = @@ -286,8 +324,14 @@ export default class MigrateAuditMailBodies extends Command { archive.push({ datasource, objects, status: 'examined', report: archived }); continue; } - // Not connected on this boot. A runtime datasource row of that name is - // an archive `os serve` connects and this boot cannot: NOT examined. + // Not connected on this boot. A declaration this boot attempted and + // did not connect, or a runtime datasource row of that name (an + // archive `os serve` connects and this boot cannot): NOT examined. + const connection = declaredConnectFailure(stack.kernel, datasource); + if (connection) { + archive.push({ datasource, objects, status: 'not-connected', connection }); + continue; + } archive.push({ datasource, objects, status: await runtimeDatasourceRowStatus(readView, datasource) }); } const archiveFailures = archive.reduce( @@ -320,12 +364,23 @@ export default class MigrateAuditMailBodies extends Command { ); } } else if (a.status === 'none') { - console.log(chalk.dim(` archive datasource '${a.datasource}': none on this deployment — nothing archived, nothing to mask`)); + console.log(chalk.dim( + ` archive datasource '${a.datasource}': none declared and connected on this boot, and no active runtime ` + + `datasource row names one — nothing examined. An archive a host registers in code outside the stack ` + + `config is not reached: declare it in the config's \`datasources\` with \`autoConnect: true\`.`, + )); } } console.log(''); for (const a of archive) { - if (a.status === 'not-connected') { + if (a.status === 'not-connected' && a.connection) { + printError( + `Archive datasource '${a.datasource}' (${a.objects.join(', ')}) was NOT examined: the stack config declares ` + + `it, and this boot did not connect it (${a.connection.status}` + + `${a.connection.reason ? `: ${a.connection.reason}` : ''}), so its archived copies may still hold a mail ` + + `body in clear. Make it reachable from where this command runs, and re-run.`, + ); + } else if (a.status === 'not-connected') { printError( `Archive datasource '${a.datasource}' (${a.objects.join(', ')}) was NOT examined: the database holds an active ` + `runtime datasource of that name, which \`os serve\` connects and this command does not, so its archived ` + From 791db71c37806719de5ba66281186385cc0a0100 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 17:16:27 +0000 Subject: [PATCH 12/12] wip(cli): pin the unreachable declared archive Claude-Session: https://claude.ai/code/session_01WQWCrorWxW5zDh4n6jCEF5 Co-authored-by: Claude --- .../audit-mail-bodies.archive.integration.test.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts b/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts index 5ae8434ce67..85f33f60386 100644 --- a/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts +++ b/packages/cli/src/commands/migrate/audit-mail-bodies.archive.integration.test.ts @@ -1,5 +1,5 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. -// test-budget: 70 — spawns the real producer once per single-cause fixture: two served-shape boots that run the real Archiver, and six spawns of the source CLI (dry run, --apply, re-run over the archive fixture; --apply on a project with no archive; a dry run with a runtime archive row; --apply with a declared archive that cannot connect); 54.7 s, the slower of two local runs on a shared container, x 1.25, no CI reading yet +// test-budget: 85 — spawns the real producer once per single-cause fixture: two served-shape boots that run the real Archiver, and six spawns of the source CLI (dry run, --apply, re-run over the archive fixture; --apply on a project with no archive; a dry run with a runtime archive row; --apply with a declared archive that cannot connect); 67.8 s, the slower of two local runs on a shared container, x 1.25, no CI reading yet /** * [#24899] `os migrate audit-mail-bodies` masks the mail-log bodies the @@ -281,12 +281,14 @@ beforeAll(async () => { } runtimeRowRun = await runCommand(runtimeRowDir, runtimeRowDb, ['--json']); - // A declared archive whose SQLite file cannot be opened: its directory is a - // regular file, so the connect fails on every attempt. + // A declared archive the boot cannot reach: a Postgres server on a port + // nothing listens on, so the connect is refused on every attempt. Not an + // unopenable SQLite file: the source entry pins NODE_ENV=development, under + // which the SQLite factory steps an unopenable file down to an in-memory + // engine instead of failing. unreachableDir = mkdtempSync(join(tmpdir(), 'os-24899-unreachable-')); - writeFileSync(join(unreachableDir, 'blocker'), 'not a directory'); writeArtifact(unreachableDir, [ - { name: 'archive', driver: 'sqlite', config: { filename: join(unreachableDir, 'blocker', 'archive.db') }, autoConnect: true }, + { name: 'archive', driver: 'postgres', config: { host: '127.0.0.1', port: 1, database: 'archive', user: 'os24899' }, autoConnect: true }, ]); unreachableRun = await runCommand(unreachableDir, join(unreachableDir, 'main.db'), ['--apply', '--yes', '--json']); }, HOOK_TIMEOUT_MS); @@ -377,8 +379,7 @@ describe('[#24899] os migrate audit-mail-bodies over an archive datasource', () expect(unreachableRun.code).toBe(1); const entry = archiveOf(unreachableRun); expect(entry).toMatchObject({ datasource: 'archive', objects: ['sys_audit_log'], status: 'not-connected' }); - expect(typeof entry.connection?.status).toBe('string'); - expect(entry.connection.status === 'connected' || entry.connection.status === 'already-registered').toBe(false); + expect(entry.connection?.status).toBe('failed-degraded'); expect(entry.report).toBeUndefined(); }); });