From b8fbc29d712576634b46188c00eac76ae7cb7a14 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 18:38:28 +0000 Subject: [PATCH 1/8] fix(packages): convert stored and installed package bodies forward at the sys_packages and marketplace doors Claude-Session: https://claude.ai/code/session_013JuCowY7RrGXYbGEv3cWsQ Co-authored-by: Claude --- .../src/marketplace-install-local-plugin.ts | 61 ++++++++++++++++--- .../services/service-package/src/index.ts | 36 ++++++++++- 2 files changed, 85 insertions(+), 12 deletions(-) diff --git a/packages/cloud-connection/src/marketplace-install-local-plugin.ts b/packages/cloud-connection/src/marketplace-install-local-plugin.ts index 7770ed0fd59..1154927185f 100644 --- a/packages/cloud-connection/src/marketplace-install-local-plugin.ts +++ b/packages/cloud-connection/src/marketplace-install-local-plugin.ts @@ -773,11 +773,18 @@ export class MarketplaceInstallLocalPlugin implements Plugin { this.reportProtocolIncompatibleEntry(ctx, entry, compat.diagnostic); continue; } + // ADR-0087 D2 — the ledger entry is data at rest: the body is + // converted forward ONCE here ({@link convertPackageBodyForward}), + // and that converted body is what registers, what the binder + // binds, what the side effects replay and what the heal seeds. + // The ledger keeps the body as installed, so it converts again + // at every rehydrate. + const converted = this.convertPackageBodyForward(ctx, entry.manifest); // Awaited: register also bridges the manifest's objects into // the metadata service (late-registration bridge in // ObjectQLPlugin) — wait for that so metadata consumers see // the package as soon as rehydrate reports success. - await manifestService!.register(entry.manifest); + await manifestService!.register(converted); // Sync schemas so the driver creates tables for the newly- // registered objects (idempotent — already-synced tables // are no-ops). @@ -787,12 +794,12 @@ export class MarketplaceInstallLocalPlugin implements Plugin { } catch { /* non-fatal */ } // [#21321] Bind the package's script-action bodies and body // hooks — `register` above makes them declared, not runnable. - await this.bindArtifactHandlers(ctx, entry.manifest, entry.manifestId); + await this.bindArtifactHandlers(ctx, converted, entry.manifestId); // Replay translations + register seed datasets, but don't // re-run seeding — existing rows are already in the DB from // the original install, and multi-tenant orgs will replay // via the security middleware on next sys_organization insert. - await this.applySideEffects(ctx, entry.manifest, { seedNow: false }); + await this.applySideEffects(ctx, converted, { seedNow: false }); // …EXCEPT when the database no longer has them: the ledger is // anchored to while the database can be swapped out from // under it (`os dev --fresh`, a deleted dev.db, a --database @@ -800,7 +807,7 @@ export class MarketplaceInstallLocalPlugin implements Plugin { // boot via AppPlugin, but a rehydrated marketplace package // would stay empty forever — app visible, tables created, // zero rows. Heal that case. - await this.maybeHealSampleData(ctx, entry); + await this.maybeHealSampleData(ctx, entry, converted); ctx.logger?.info?.(`[MarketplaceInstallLocal] rehydrated ${entry.manifestId}@${entry.version}`); } catch (err: any) { ctx.logger?.error?.(`[MarketplaceInstallLocal] rehydrate failed for ${entry.manifestId}`, err instanceof Error ? err : new Error(String(err))); @@ -821,11 +828,15 @@ export class MarketplaceInstallLocalPlugin implements Plugin { * install-time rows (or the user's own data) are still present, and * re-upserting would silently revert user edits on every boot. * + * `converted` is the entry's manifest as rehydrate converted it — the one the + * package registered — so the heal seeds what registered, not the raw + * ledger bytes. + * * Never throws — a failed heal logs and leaves the boot untouched. */ - private maybeHealSampleData = async (ctx: PluginContext, entry: InstalledEntry): Promise => { - const datasets = Array.isArray(entry.manifest?.data) - ? entry.manifest.data.filter((d: any) => d && d.object && Array.isArray(d.records)) + private maybeHealSampleData = async (ctx: PluginContext, entry: InstalledEntry, converted: any): Promise => { + const datasets = Array.isArray(converted?.data) + ? converted.data.filter((d: any) => d && d.object && Array.isArray(d.records)) : []; if (datasets.length === 0) return; if (entry.sampleDataPurged === true) return; @@ -1546,12 +1557,18 @@ export class MarketplaceInstallLocalPlugin implements Plugin { // running kernel held a package it did not, and the console read // that answer as "the app should now appear". let hotLoadError: string | undefined; + // ADR-0087 D2 — the fetched bundle is a published artifact, so it is + // converted forward ONCE here ({@link convertPackageBodyForward}); the + // converted body is what registers, what the binder binds (4c) and + // what the side effects replay (5). The ledger entry (4) keeps the + // body as installed, and rehydrate converts it again at every boot. + const converted = this.convertPackageBodyForward(ctx, manifest); try { const manifestService = ctx.getService('manifest') as any; // Awaited: register also bridges the manifest's objects into the // metadata service — a caller that reads metadata right after a // 200 (AI describe_object, Studio object list) must see them. - await manifestService.register(manifest); + await manifestService.register(converted); } catch (err: any) { const registerError = String(err?.message ?? err); // For offline file imports we treat a register failure as a hard @@ -1615,14 +1632,14 @@ export class MarketplaceInstallLocalPlugin implements Plugin { // through the runtime's ONE binder — the call `AppPlugin.start` // makes for a boot artifact. A reinstall replaces the previous // version's set rather than adding to it. - await this.bindArtifactHandlers(ctx, manifest, manifestId); + await this.bindArtifactHandlers(ctx, converted, manifestId); // 5. Replicate the AppPlugin start-time side-effects that the // `manifest` service does NOT do on its own: // • load translation bundles into the i18n service // • stash seed datasets on the kernel + run them now so the // installed app has demo data on first paint. - const seededSummary = await this.applySideEffects(ctx, manifest, { seedNow: true, c }); + const seededSummary = await this.applySideEffects(ctx, converted, { seedNow: true, c }); // `skipped` counts too: an all-skip run means the rows are ALREADY in // the database (e.g. a reinstall/upgrade over live sample data) — the // install carries sample data either way. Leaving the flag false here @@ -2752,6 +2769,30 @@ export class MarketplaceInstallLocalPlugin implements Plugin { }, refusal.status); }; + /** + * ADR-0087 D2 for this plugin's two loads of a ledger body — the install + * and the rehydrate. The body is data at rest (a published artifact), so it + * replays the versioned forward conversion inside the window its own + * `engines.protocol` opens, through the engine's ONE registration-door entry + * point (`ObjectQL.convertPackageBodyForward`), reused here, never + * re-derived: a body with no declared range comes back by reference, and the + * named warning is the engine's own. + * + * Converted once per load and handed to EVERY reader of that load — + * `manifest.register`, {@link bindArtifactHandlers}, {@link applySideEffects} + * and the sample-data heal — so none of them reads the raw bytes while the + * registry holds the converted copy. `register` meets the same entry point + * again, which hands an already-converted body back by reference with no + * second warning. + * + * An engine without that door (not ObjectQL) is handed the body as before. + */ + private convertPackageBodyForward = (ctx: PluginContext, manifest: unknown): any => { + let ql: { convertPackageBodyForward?: (body: unknown) => unknown } | undefined; + try { ql = ctx.getService('objectql') as typeof ql; } catch { /* no data engine */ } + return typeof ql?.convertPackageBodyForward === 'function' ? ql.convertPackageBodyForward(manifest) : manifest; + }; + /** * [#21321] Bind an installed package's executable handlers — its * `type: 'script'` action bodies and its body hooks — through diff --git a/packages/services/service-package/src/index.ts b/packages/services/service-package/src/index.ts index 9c6cfa75f2b..ce61da9f267 100644 --- a/packages/services/service-package/src/index.ts +++ b/packages/services/service-package/src/index.ts @@ -678,7 +678,25 @@ export class PackageServicePlugin implements Plugin { // from the filesystem. Best-effort and non-fatal. try { const registry = (objectql as unknown as { registry?: any }).registry; + // ADR-0087 D2 — a stored record is data at rest, so it replays the + // versioned forward conversion inside the window its own + // `engines.protocol` opens, BEFORE the registry judges it. The judge of + // that window is the engine's ONE registration-door entry point + // (`ObjectQL.convertPackageBodyForward`), reused here, never re-derived: + // a record with no declared range comes back by reference and meets the + // registry's refusals exactly as before, and the conversion's named + // warning is the engine's own. The stored row is not rewritten, so it + // converts again at every boot (the ADR-0087 addendum: metadata at rest + // replays the chain at every load). Reached through the engine this + // plugin already holds, as `registry` is; an engine with no such door + // (not ObjectQL) is handed the stored body as before. + const convertForward = (objectql as unknown as { + convertPackageBodyForward?: (body: unknown) => unknown; + }).convertPackageBodyForward?.bind(objectql); if (registry?.installPackage && registry?.getPackage) { + // Counts the records the registry ACCEPTED in this pass — not the ones + // the handshake below refuses, not the ones already registered, and + // not the ones the registry refuses (each of those is reported by name). let hydrated = 0; for (const rec of await packageService.list()) { const id = rec?.manifest?.id; @@ -712,8 +730,22 @@ export class PackageServicePlugin implements Plugin { `'${compat.requiredRange}'; skipping the protocol handshake (ADR-0087).`, ); } - registry.installPackage(rec.manifest); - hydrated++; + // One record's refusal is that record's: it is reported by name and + // the records after it keep hydrating. It used to throw out of the + // whole loop into the catch below, which logged it at `debug` — + // and every durable package after the bad one was silently absent. + try { + registry.installPackage(convertForward ? convertForward(rec.manifest) : rec.manifest); + hydrated++; + } catch (refusal) { + logger.error( + `Package '${id}' from sys_packages was NOT hydrated into the registry — the registry refused its ` + + `stored body: ${(refusal as Error)?.message ?? String(refusal)} It is absent from the registry for ` + + `this process while its sys_packages row remains; the other packages hydrate as usual. Install a ` + + `corrected build of '${id}' or uninstall it, then restart.`, + refusal as Error, + ); + } } } if (hydrated > 0) { From 155b9245e9c4600a359b2d909c0032e67a1f14d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 18:42:59 +0000 Subject: [PATCH 2/8] test(service-package): pin sys_packages hydration's forward conversion over the real engine and sqlite Claude-Session: https://claude.ai/code/session_013JuCowY7RrGXYbGEv3cWsQ Co-authored-by: Claude --- .../services/service-package/package.json | 2 + .../src/hydration-convert-forward.test.ts | 222 ++++++++++++++++++ .../services/service-package/vitest.config.ts | 21 +- pnpm-lock.yaml | 6 + 4 files changed, 247 insertions(+), 4 deletions(-) create mode 100644 packages/services/service-package/src/hydration-convert-forward.test.ts diff --git a/packages/services/service-package/package.json b/packages/services/service-package/package.json index 2f76a90d018..d25dba5aa4f 100644 --- a/packages/services/service-package/package.json +++ b/packages/services/service-package/package.json @@ -29,6 +29,8 @@ "@objectstack/metadata-core": "workspace:*" }, "devDependencies": { + "@objectstack/driver-sql": "workspace:*", + "@objectstack/objectql": "workspace:*", "@types/node": "^26.6.3", "typescript": "^6.0.3", "vitest": "^4.1.11" diff --git a/packages/services/service-package/src/hydration-convert-forward.test.ts b/packages/services/service-package/src/hydration-convert-forward.test.ts new file mode 100644 index 00000000000..4c4170a0656 --- /dev/null +++ b/packages/services/service-package/src/hydration-convert-forward.test.ts @@ -0,0 +1,222 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The `sys_packages` boot hydration is a LOAD of data at rest, so a stored + * package body that declares its protocol range converts forward under ADR-0087 + * D2 before the registry judges it — through the engine's one registration-door + * entry point (`ObjectQL.convertPackageBodyForward`), reused, never re-derived. + * + * Over the REAL stack, end to end: an `ObjectKernel` with `ObjectQLPlugin` (the + * real `SchemaRegistry`, which is the judge that refuses) over a real + * `SqlDriver` (better-sqlite3, on disk), and this package's plugin. One process + * publishes the records through the `package` service; a second process over + * the same database file boots and hydrates them, as a restart does. A stubbed + * registry cannot refuse anything, so it could not tell these cases apart. + * + * ## Which bodies reach the window on this runtime + * + * The body is HotCRM 4.0.1's (`__fixtures__/hotcrm-4.0.1-published-pages.artifact.json` + * in `@objectstack/objectql`): a `type: 'home'` page with `isDefault: true`, + * declared floor 17.7.0, a shape this spec refuses since 18.0. As PUBLISHED it + * declares `^17.7.0`, a range that excludes this runtime's major, so the ADR-0087 + * D1 handshake that runs first refuses it by name and it never reaches the + * registry (pinned below as the fact it is). The window opens for a stored body + * whose declared range ADMITS this runtime with a floor before the retirement, + * so the converting cases restamp the same body `>=17.7.0` — same floor, same + * pages, a range the handshake accepts. + * + * What is pinned: + * - that body hydrates, converted, with the engine's named warning; the + * registry serves the converted record and the stored row is left as + * published; + * - one record the registry still refuses after conversion is reported BY + * NAME at `error`, with the refusal attached, and the record listed after + * it keeps hydrating — it used to throw out of the whole loop to a `debug` + * line; + * - CONTROL: the same body with no declared range (a manifest built in code) + * meets the registry's refusal exactly as before, and nothing converts; + * - HotCRM 4.0.1 exactly as published is refused by the handshake, not the + * registry, and the records after it keep hydrating. + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { ObjectKernel } from '@objectstack/core'; +import type { Plugin, PluginContext } from '@objectstack/core'; +import { ObjectQLPlugin, type ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { PackageServicePlugin, type PackageService } from './index.js'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const FIXTURE_PATH = resolve(HERE, '../../../objectql/src/__fixtures__/hotcrm-4.0.1-published-pages.artifact.json'); + +type Bag = Record; +type Line = { level: 'info' | 'warn' | 'error'; text: string; error?: unknown }; + +const HOTCRM_ID = 'app.objectstack.hotcrm'; +/** The same body as a plugin would build it in code: its own id, no namespace, NO declared range. */ +const CODE_BUILT_ID = 'com.example.code_built_home'; +const IS_DEFAULT_CONVERSION = 'page-is-default-off-record-page-removed'; + +/** HotCRM 4.0.1's `app.objectstack.hotcrm` body, flat, exactly as `os build` assembled it (`^17.7.0`). */ +const publishedHotcrmBody = (): Bag => { + const artifact = JSON.parse(readFileSync(FIXTURE_PATH, 'utf8')); + return artifact.packages.find((e: Bag) => e.manifest.id === HOTCRM_ID).manifest; +}; +/** The same body under a range that admits this runtime, from the same 17.7.0 floor. */ +const hotcrmBody = (): Bag => ({ ...publishedHotcrmBody(), engines: { protocol: '>=17.7.0' } }); +const codeBuiltBody = (): Bag => { + const { engines: _engines, namespace: _namespace, ...rest } = hotcrmBody(); + return { ...rest, id: CODE_BUILT_ID, name: 'Code-built home' }; +}; + +const dirs: string[] = []; +const kernels: ObjectKernel[] = []; +afterEach(async () => { + while (kernels.length) { + const k = kernels.pop()!; + if (k.getState() === 'running') await k.shutdown(); + } + while (dirs.length) rmSync(dirs.pop()!, { recursive: true, force: true }); +}); + +/** + * One process over the database file in `dir`. A first plugin registers the + * driver and records every line the engine and the plugins log, from before + * the package service's `start()` — which is where the hydration runs. + */ +async function boot(dir: string) { + const lines: Line[] = []; + const record = (target: Bag | undefined) => { + if (!target) return; + for (const level of ['info', 'warn', 'error'] as const) { + target[level] = (text: unknown, error?: unknown) => { lines.push({ level, text: String(text), error }); }; + } + }; + const driver = new SqlDriver({ + client: 'better-sqlite3', + connection: { filename: join(dir, 'data.sqlite') }, + useNullAsDefault: true, + }); + const recorder: Plugin = { + name: 'test.sqlite-and-log-recorder', + version: '1.0.0', + async init(ctx: PluginContext) { + ctx.getService('objectql').registerDriver(driver as never, true); + }, + async start(ctx: PluginContext) { + record(ctx.logger as unknown as Bag); + record((ctx.getService('objectql') as unknown as { logger?: Bag }).logger); + }, + }; + const kernel = new ObjectKernel({ logger: { level: 'silent' }, gracefulShutdown: false } as never); + kernels.push(kernel); + await kernel.use(new ObjectQLPlugin()); + await kernel.use(recorder); + await kernel.use(new PackageServicePlugin()); + await kernel.bootstrap(); + const ql = kernel.getService('objectql'); + ql.registry.logLevel = 'silent'; + return { kernel, ql, lines, packages: kernel.getService('package') }; +} + +/** + * Publish `bodies` in a first process, listed in the order given (the hydration + * walks `sys_packages` newest first, so each row is stamped older than the one + * before it), and stop it — then boot the second process, which hydrates. + */ +async function restartWith(bodies: Bag[]) { + const dir = mkdtempSync(join(tmpdir(), 'os-sys-packages-hydration-')); + dirs.push(dir); + const first = await boot(dir); + for (const [i, manifest] of bodies.entries()) { + expect(await first.packages.publish({ manifest: manifest as never, metadata: {} as never })).toEqual({ success: true }); + await first.ql.execute({ + sql: 'UPDATE sys_packages SET created_at = ? WHERE id = ?', + args: [`2026-01-01 00:00:${String(59 - i).padStart(2, '0')}`, manifest.id], + }); + } + expect((await first.packages.list()).map((r: Bag) => r.manifest.id)).toEqual(bodies.map((b) => b.id)); + await first.kernel.shutdown(); + return boot(dir); +} + +const named = (lines: Line[], id: string) => + lines.filter((l) => l.level === 'warn' && l.text.includes(`package '${id}'`) && l.text.includes('ADR-0087 conversion')); +const notHydrated = (lines: Line[], id: string) => + lines.filter((l) => l.level === 'error' && l.text.startsWith(`Package '${id}' from sys_packages was NOT hydrated`)); + +const expectMisplacedDefault = (line: Line | undefined, id: string) => { + expect(line, `no error line names ${id}`).toBeDefined(); + const refusal = line!.error as Error & { code?: string; status?: number }; + expect({ name: refusal.name, code: refusal.code, status: refusal.status }) + .toEqual({ name: 'MisplacedDefaultPageError', code: 'INVALID_METADATA', status: 422 }); + expect(line!.text).toContain(`Page "sales_home_page" of package "${id}": \`isDefault: true\` is refused on a \`type: 'home'\` page`); +}; + +describe('sys_packages boot hydration converts a stored body forward before the registry judges it', () => { + it('hydrates HotCRM 4.0.1\'s stored body, converted, with the engine\'s named warning', async () => { + const { ql, lines, packages } = await restartWith([hotcrmBody()]); + + const record = ql.registry.getPackage(HOTCRM_ID); + expect(record, 'the stored HotCRM body was not hydrated').toBeDefined(); + const home = ((record!.manifest as Bag).pages as Bag[]).find((p) => p.name === 'sales_home_page')!; + expect(home).toMatchObject({ type: 'home' }); + expect(home).not.toHaveProperty('isDefault'); + + const warning = named(lines, HOTCRM_ID).filter((l) => l.text.includes(`conversion '${IS_DEFAULT_CONVERSION}'`)); + expect(warning).toHaveLength(1); + expect(warning[0].text).toContain('declared protocol floor 17.7.0'); + expect(notHydrated(lines, HOTCRM_ID)).toEqual([]); + expect(lines.filter((l) => l.level === 'info' && l.text === 'Hydrated 1 package(s) from sys_packages into registry')).toHaveLength(1); + + // The row stays as published: data at rest replays the chain at every load. + const [row] = await packages.list(); + expect(((row.manifest as Bag).pages as Bag[]).find((p) => p.name === 'sales_home_page')).toMatchObject({ isDefault: true }); + }); + + it('reports a record the registry still refuses by name, and keeps hydrating the records after it', async () => { + const { ql, lines } = await restartWith([codeBuiltBody(), hotcrmBody()]); + + // The refused record: named, at error, with the registry's own refusal attached. + const [line, ...more] = notHydrated(lines, CODE_BUILT_ID); + expect(more).toEqual([]); + expectMisplacedDefault(line, CODE_BUILT_ID); + expect(ql.registry.getPackage(CODE_BUILT_ID)).toBeUndefined(); + + // The record listed after it still hydrated, converted. + expect(ql.registry.getPackage(HOTCRM_ID)).toBeDefined(); + expect(named(lines, HOTCRM_ID).length).toBeGreaterThan(0); + // The count is what the registry accepted. + expect(lines.filter((l) => l.level === 'info' && l.text.startsWith('Hydrated '))).toEqual([ + expect.objectContaining({ text: 'Hydrated 1 package(s) from sys_packages into registry' }), + ]); + }); + + it('CONTROL — a stored body with no declared range meets the registry\'s refusal exactly as before, unconverted', async () => { + const { ql, lines } = await restartWith([codeBuiltBody()]); + + expectMisplacedDefault(notHydrated(lines, CODE_BUILT_ID)[0], CODE_BUILT_ID); + expect(ql.registry.getPackage(CODE_BUILT_ID)).toBeUndefined(); + expect(lines.filter((l) => l.text.includes('ADR-0087 conversion'))).toEqual([]); + expect(lines.filter((l) => l.level === 'info' && l.text.startsWith('Hydrated '))).toEqual([]); + }); + + it('HotCRM 4.0.1 exactly as published (`^17.7.0`) is refused by the D1 handshake, named, and the records after it keep hydrating', async () => { + const other = { id: 'com.example.after', name: 'After', version: '1.0.0', type: 'app' }; + const { ql, lines } = await restartWith([publishedHotcrmBody(), other]); + + const refused = lines.filter((l) => l.level === 'error' + && l.text.startsWith(`[protocol] refusing to rehydrate package '${HOTCRM_ID}' from sys_packages`)); + expect(refused).toHaveLength(1); + expect(refused[0].text).toContain('OS_PROTOCOL_INCOMPATIBLE'); + expect(ql.registry.getPackage(HOTCRM_ID)).toBeUndefined(); + // The handshake answered first: nothing converted and the registry never judged it. + expect(named(lines, HOTCRM_ID)).toEqual([]); + expect(notHydrated(lines, HOTCRM_ID)).toEqual([]); + expect(ql.registry.getPackage('com.example.after')).toBeDefined(); + }); +}); diff --git a/packages/services/service-package/vitest.config.ts b/packages/services/service-package/vitest.config.ts index 5e0591efc44..fbafaeec329 100644 --- a/packages/services/service-package/vitest.config.ts +++ b/packages/services/service-package/vitest.config.ts @@ -1,10 +1,11 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. -// This config exists for exactly one setting; everything else stays on -// vitest's defaults, deliberately — a key added here re-specifies behaviour -// for every test file in the package (packages/cli/vitest.config.ts's header -// records the incident that taught that). +// This config exists for two settings; everything else stays on vitest's +// defaults, deliberately — a key added here re-specifies behaviour for every +// test file in the package (packages/cli/vitest.config.ts's header records the +// incident that taught that). import { defineConfig } from 'vitest/config'; +import path from 'node:path'; export default defineConfig({ test: { @@ -16,4 +17,16 @@ export default defineConfig({ // Enforced repo-wide by scripts/check-console-intercept-disarm.mjs. disableConsoleIntercept: true, }, + resolve: { + // `hydration-convert-forward.test.ts` boots the REAL engine over a REAL + // driver, so its verdict must be about the source in this checkout, never + // a sibling's `dist/` (`pnpm check:test-source-alias`). ANCHORED regex, + // array form: a bare string `find` matches by PREFIX and would swallow a + // subpath into `…/src/index.ts/`. + alias: [ + { find: /^@objectstack\/core$/, replacement: path.resolve(__dirname, '../../core/src/index.ts') }, + { find: /^@objectstack\/objectql$/, replacement: path.resolve(__dirname, '../../objectql/src/index.ts') }, + { find: /^@objectstack\/driver-sql$/, replacement: path.resolve(__dirname, '../../drivers/driver-sql/src/index.ts') }, + ], + }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 14765ac91c5..5a641003bd5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2824,6 +2824,12 @@ importers: specifier: workspace:* version: link:../../spec devDependencies: + '@objectstack/driver-sql': + specifier: workspace:* + version: link:../../drivers/driver-sql + '@objectstack/objectql': + specifier: workspace:* + version: link:../../objectql '@types/node': specifier: ^26.6.3 version: 26.6.3 From 9c2d44e82ae157be9faff51387373d925360cd2c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 18:45:28 +0000 Subject: [PATCH 3/8] test(cloud-connection): pin that the marketplace install's readers share one converted body Claude-Session: https://claude.ai/code/session_013JuCowY7RrGXYbGEv3cWsQ Co-authored-by: Claude --- ...lace-install-local-convert-forward.test.ts | 241 ++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100644 packages/cloud-connection/src/marketplace-install-local-convert-forward.test.ts diff --git a/packages/cloud-connection/src/marketplace-install-local-convert-forward.test.ts b/packages/cloud-connection/src/marketplace-install-local-convert-forward.test.ts new file mode 100644 index 00000000000..16e2b066ae9 --- /dev/null +++ b/packages/cloud-connection/src/marketplace-install-local-convert-forward.test.ts @@ -0,0 +1,241 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The marketplace install's two loads of a package body — the install and the + * boot rehydrate — convert the body forward ONCE (ADR-0087 D2) and hand that + * one converted body to every reader of the load: `manifest.register`, the + * handler binder (`bindArtifactHandlers`), the side effects + * (`applySideEffects`: translations and seed datasets) and the rehydrate's + * sample-data heal. Before, `register` converted its own copy while the three + * side readers read the raw ledger entry. + * + * The converter and the judge are REAL: an `ObjectKernel` with `ObjectQLPlugin` + * supplies the engine's `convertPackageBodyForward` and the `manifest` service + * over the real `SchemaRegistry` (a stubbed registry cannot refuse, so it could + * not tell the CONTROL below from the converting case). The side readers are + * replaced by recorders, because what is pinned is WHICH body they receive. + * + * The body is HotCRM 4.0.1's (`__fixtures__/hotcrm-4.0.1-published-pages.artifact.json` + * in `@objectstack/objectql`): a `type: 'home'` page with `isDefault: true`, + * declared floor 17.7.0. As published it declares `^17.7.0`, which this + * runtime's ADR-0087 D1 handshake refuses before any load (the protocol + * handshake suite pins that), so the converting cases restamp the same body + * `>=17.7.0`: same floor, same pages, a range this runtime admits. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +// The plugin reaches `@objectstack/runtime` through a lazy `import()`; its first +// load is paid here, at module top, never inside a clocked `it`. +import '@objectstack/runtime'; +import { ObjectKernel } from '@objectstack/core'; +import { ObjectQLPlugin, type ObjectQL } from '@objectstack/objectql'; +import { MarketplaceInstallLocalPlugin } from './marketplace-install-local-plugin.js'; +import { LocalManifestSource } from './local-manifest-source.js'; +import { installerAuthService, withInstallerGrants } from './install-local-principal.fixtures.js'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const FIXTURE_PATH = resolve(HERE, '../../objectql/src/__fixtures__/hotcrm-4.0.1-published-pages.artifact.json'); + +type Bag = Record; +type Handler = (c: any) => Promise; + +const HOTCRM_ID = 'app.objectstack.hotcrm'; +const IS_DEFAULT_CONVERSION = 'page-is-default-off-record-page-removed'; +const ROUTE = '/api/v1/marketplace/install-local'; + +/** HotCRM 4.0.1's body under a range this runtime admits, from the same 17.7.0 floor. */ +const hotcrmBody = (): Bag => { + const artifact = JSON.parse(readFileSync(FIXTURE_PATH, 'utf8')); + const body = artifact.packages.find((e: Bag) => e.manifest.id === HOTCRM_ID).manifest; + return { ...body, engines: { protocol: '>=17.7.0' } }; +}; +/** The same body as a plugin builds it in code: NO declared range. */ +const codeBuiltBody = (): Bag => { + const { engines: _engines, ...rest } = hotcrmBody(); + return rest; +}; +const homePage = (body: Bag) => (body.pages as Bag[]).find((p) => p.name === 'sales_home_page'); + +let dir: string; +const kernels: ObjectKernel[] = []; +beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'mil-convert-forward-')); }); +afterEach(async () => { + while (kernels.length) { + const k = kernels.pop()!; + if (k.getState() === 'running') await k.shutdown(); + } + rmSync(dir, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +function seedLedger(manifest: Bag): void { + new LocalManifestSource(dir).write({ + packageId: HOTCRM_ID, + versionId: 'local', + manifestId: HOTCRM_ID, + version: manifest.version, + manifest, + installedAt: '2026-01-01T00:00:00.000Z', + installedBy: 'admin', + withSampleData: false, + }); +} + +/** + * The plugin over a REAL engine's converter and `manifest` service, booted to + * `kernel:ready` (which runs the rehydrate). Every reader of a load records the + * body it was handed, in one ordered list. + */ +async function bootPlugin() { + const kernel = new ObjectKernel({ logger: { level: 'silent' }, gracefulShutdown: false } as never); + kernels.push(kernel); + await kernel.use(new ObjectQLPlugin()); + await kernel.bootstrap(); + const ql = kernel.getService('objectql'); + ql.registry.logLevel = 'silent'; + const conversionWarnings: string[] = []; + (ql as unknown as { logger: { warn: (line: unknown) => void } }).logger.warn = (line: unknown) => { + if (String(line).includes('ADR-0087 conversion')) conversionWarnings.push(String(line)); + }; + + const handed: Array<{ reader: string; body: Bag }> = []; + const realManifest = kernel.getService<{ register(m: unknown): unknown }>('manifest'); + const manifest = { + register: async (body: Bag) => { + // The plugin also registers its own Setup UI bundle at + // `kernel:ready`; only the package under test is recorded. + if (body?.id === HOTCRM_ID) handed.push({ reader: 'register', body }); + await realManifest.register(body); + }, + }; + const routes = new Map(); + const rawApp = { + get: (p: string, h: Handler) => routes.set(`GET ${p}`, h), + post: (p: string, h: Handler) => routes.set(`POST ${p}`, h), + delete: (p: string, h: Handler) => routes.set(`DELETE ${p}`, h), + }; + const hooks = new Map Promise>(); + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }; + const services: Record = { + manifest, + auth: installerAuthService(), + // The engine's own door, bound to the real engine; the rest of the + // surface this plugin reads off `objectql` is the registry's. + objectql: withInstallerGrants({ + convertPackageBodyForward: (body: unknown) => ql.convertPackageBodyForward(body), + registry: ql.registry, + syncSchemas: async () => undefined, + }), + }; + const ctx = { + hook: (e: string, h: () => Promise) => hooks.set(e, h), + getService: (name: string) => { + if (name === 'http-server') return { getRawApp: () => rawApp }; + if (!(name in services)) throw new Error(`no ${name}`); + return services[name]; + }, + logger, + }; + + const plugin = new MarketplaceInstallLocalPlugin({ controlPlaneUrl: 'off', storageDir: dir }); + const recorder = (reader: string) => async (...args: unknown[]) => { + handed.push({ reader, body: (reader === 'heal' ? args[2] : args[1]) as Bag }); + return { seeded: { mode: 'none' } }; + }; + const p = plugin as unknown as Record; + p.bindArtifactHandlers = recorder('bindArtifactHandlers'); + p.applySideEffects = recorder('applySideEffects'); + p.maybeHealSampleData = recorder('heal'); + + await plugin.start(ctx as never); + await hooks.get('kernel:ready')?.(); + + const install = async (body: Bag) => { + const json = vi.fn((payload: any, status?: number) => ({ payload, status: status ?? 200 })); + return routes.get(`POST ${ROUTE}`)!({ + req: { + url: `http://localhost:3000${ROUTE}`, + raw: new Request('http://localhost:3000/x'), + json: async () => ({ manifest: body }), + param: () => undefined, + }, + json, + }); + }; + return { ql, handed, conversionWarnings, logger, install }; +} + +/** Every reader received the SAME converted object, and the flag is gone from it. */ +function expectOneConvertedBody(handed: Array<{ reader: string; body: Bag }>, readers: string[]) { + expect(handed.map((h) => h.reader)).toEqual(readers); + const [first] = handed; + for (const h of handed) expect(h.body, `${h.reader} read a different body`).toBe(first.body); + expect(homePage(first.body)).toMatchObject({ type: 'home' }); + expect(homePage(first.body)).not.toHaveProperty('isDefault'); +} + +describe('the marketplace install converts a body once and every reader of the load reads that body', () => { + it('rehydrate — register, the binder, the side effects and the heal all receive the converted body', async () => { + seedLedger(hotcrmBody()); + const { ql, handed, conversionWarnings } = await bootPlugin(); + + expectOneConvertedBody(handed, ['register', 'bindArtifactHandlers', 'applySideEffects', 'heal']); + expect(ql.registry.getPackage(HOTCRM_ID)).toBeDefined(); + // Converted once: `register` met the entry point again with the + // converted body, and no second warning was printed. + expect(conversionWarnings.filter((w) => w.includes(`conversion '${IS_DEFAULT_CONVERSION}'`))).toHaveLength(1); + // The ledger keeps the body as installed: data at rest replays at every load. + expect(homePage(new LocalManifestSource(dir).read(HOTCRM_ID).entry!.manifest as Bag)).toMatchObject({ isDefault: true }); + }); + + it('install — register, the binder and the side effects all receive the converted body', async () => { + const { ql, handed, conversionWarnings, install } = await bootPlugin(); + + const res = await install(hotcrmBody()); + + expect(res.status, JSON.stringify(res.payload)).toBe(200); + expectOneConvertedBody(handed, ['register', 'bindArtifactHandlers', 'applySideEffects']); + expect(ql.registry.getPackage(HOTCRM_ID)).toBeDefined(); + expect(conversionWarnings.filter((w) => w.includes(`conversion '${IS_DEFAULT_CONVERSION}'`))).toHaveLength(1); + expect(homePage(new LocalManifestSource(dir).read(HOTCRM_ID).entry!.manifest as Bag)).toMatchObject({ isDefault: true }); + }); +}); + +describe('CONTROL — a body with no declared range meets the registry\'s refusal exactly as before', () => { + const expectMisplacedDefault = (message: string) => + expect(message).toContain(`Page "sales_home_page" of package "${HOTCRM_ID}": \`isDefault: true\` is refused on a \`type: 'home'\` page`); + + it('install — refused at register (422 PLUGIN_REGISTER_FAILED), nothing converted, no side reader runs', async () => { + const { ql, handed, conversionWarnings, install } = await bootPlugin(); + + const res = await install(codeBuiltBody()); + + expect(res.status).toBe(422); + expect(res.payload.error.code).toBe('PLUGIN_REGISTER_FAILED'); + expectMisplacedDefault(res.payload.error.message); + expect(handed.map((h) => h.reader)).toEqual(['register']); + expect(homePage(handed[0].body)).toMatchObject({ isDefault: true }); + expect(conversionWarnings).toEqual([]); + expect(ql.registry.getPackage(HOTCRM_ID)).toBeUndefined(); + }); + + it('rehydrate — refused at register and reported for that entry, no side reader runs', async () => { + seedLedger(codeBuiltBody()); + const { ql, handed, conversionWarnings, logger } = await bootPlugin(); + + expect(handed.map((h) => h.reader)).toEqual(['register']); + expect(homePage(handed[0].body)).toMatchObject({ isDefault: true }); + expect(conversionWarnings).toEqual([]); + expect(ql.registry.getPackage(HOTCRM_ID)).toBeUndefined(); + const failed = logger.error.mock.calls.filter((c: unknown[]) => String(c[0]) === `[MarketplaceInstallLocal] rehydrate failed for ${HOTCRM_ID}`); + expect(failed).toHaveLength(1); + const refusal = failed[0][1] as Error & { code?: string; status?: number }; + expect({ name: refusal.name, code: refusal.code, status: refusal.status }) + .toEqual({ name: 'MisplacedDefaultPageError', code: 'INVALID_METADATA', status: 422 }); + expectMisplacedDefault(refusal.message); + }); +}); From 9ec68c6b40e0f13d1855a7fa8def26b13715bad6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 18:47:50 +0000 Subject: [PATCH 4/8] fix(metadata-protocol): record why the install primitive stays strict under ADR-0087 D2, pinned over the real registry Claude-Session: https://claude.ai/code/session_013JuCowY7RrGXYbGEv3cWsQ Co-authored-by: Claude --- packages/metadata-protocol/src/protocol.ts | 13 ++++++ .../services/service-package/package.json | 1 + .../src/hydration-convert-forward.test.ts | 41 +++++++++++++++++++ .../services/service-package/vitest.config.ts | 3 +- pnpm-lock.yaml | 3 ++ scripts/cross-package-test-inputs.mjs | 14 +++++++ turbo.json | 13 ++++++ 7 files changed, 87 insertions(+), 1 deletion(-) diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 74fb23d7fcf..2d3ae80960c 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -31390,6 +31390,19 @@ export class ObjectStackProtocolImplementation implements // install REPLACES the row object (and may register the namespace). const undoInstall = this.packageInstallUndo(manifest); + // ADR-0087 D2 — this door is AUTHORING-side, so the registry judges the + // manifest exactly as handed, and no forward conversion runs here (the + // window the registration door, the `sys_packages` hydration and the + // marketplace install open is for bodies at rest). Its callers hand it + // a manifest being written now: `POST /api/v1/packages` parses the + // whole body through the closed `ManifestSchema` first, which refuses + // a metadata collection such as `pages` outright; `duplicatePackage` + // copies the registry's own record, already judged; the unmanaged + // install's base carries identity keys only; and AI Studio's blueprint + // tools author in-process. A retired declaration there is the author's + // to fix, refused loudly at the moment of writing. Once the body sits + // in `sys_packages` it is data at rest, and the boot hydration + // (`@objectstack/service-package`) converts it at every load. let pkg = this.engine.registry.installPackage(manifest as any, request.settings); // [#19277] HONOUR `enableOnInstall` — the key THIS request contract diff --git a/packages/services/service-package/package.json b/packages/services/service-package/package.json index d25dba5aa4f..3a406e6d992 100644 --- a/packages/services/service-package/package.json +++ b/packages/services/service-package/package.json @@ -30,6 +30,7 @@ }, "devDependencies": { "@objectstack/driver-sql": "workspace:*", + "@objectstack/metadata-protocol": "workspace:*", "@objectstack/objectql": "workspace:*", "@types/node": "^26.6.3", "typescript": "^6.0.3", diff --git a/packages/services/service-package/src/hydration-convert-forward.test.ts b/packages/services/service-package/src/hydration-convert-forward.test.ts index 4c4170a0656..8070ce61833 100644 --- a/packages/services/service-package/src/hydration-convert-forward.test.ts +++ b/packages/services/service-package/src/hydration-convert-forward.test.ts @@ -37,6 +37,20 @@ * meets the registry's refusal exactly as before, and nothing converts; * - HotCRM 4.0.1 exactly as published is refused by the handshake, not the * registry, and the records after it keep hydrating. + * + * ## The install door stays strict, and why (`protocol.installPackage`) + * + * The same converting body handed to the install primitive is REFUSED by the + * registry, and nothing lands in `sys_packages`. That door is authoring-side: + * its HTTP caller, `POST /api/v1/packages`, parses the whole body through the + * closed `ManifestSchema` first, which refuses a `pages` key outright, so no + * package body carrying metadata collections reaches the primitive over HTTP; + * `duplicatePackage` copies the registry's own record (already judged), and + * the unmanaged install's base is identity keys only. What it does receive is + * a manifest being authored now (AI Studio's blueprint tools call it + * in-process), and a retired declaration there is the author's to fix, loudly, + * at the moment of writing. A body that lands in `sys_packages` by any road is + * data at rest from then on, and the hydration above converts it. */ import { describe, it, expect, afterEach } from 'vitest'; @@ -48,6 +62,7 @@ import { ObjectKernel } from '@objectstack/core'; import type { Plugin, PluginContext } from '@objectstack/core'; import { ObjectQLPlugin, type ObjectQL } from '@objectstack/objectql'; import { SqlDriver } from '@objectstack/driver-sql'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; import { PackageServicePlugin, type PackageService } from './index.js'; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -157,6 +172,32 @@ const expectMisplacedDefault = (line: Line | undefined, id: string) => { expect(line!.text).toContain(`Page "sales_home_page" of package "${id}": \`isDefault: true\` is refused on a \`type: 'home'\` page`); }; +describe('the install primitive is authoring-side and stays strict', () => { + it('protocol.installPackage refuses the same body with the registry\'s refusal, and writes no sys_packages row', async () => { + const dir = mkdtempSync(join(tmpdir(), 'os-sys-packages-install-')); + dirs.push(dir); + const { ql, lines, packages } = await boot(dir); + const protocol = new ObjectStackProtocolImplementation( + ql as never, + () => new Map([['package', packages]]), + ); + + let refusal: (Error & { code?: string; status?: number }) | undefined; + try { + await protocol.installPackage({ manifest: hotcrmBody() } as never); + } catch (e) { + refusal = e as typeof refusal; + } + expect(refusal, 'the install primitive accepted a retired declaration').toBeDefined(); + expect({ name: refusal!.name, code: refusal!.code, status: refusal!.status }) + .toEqual({ name: 'MisplacedDefaultPageError', code: 'INVALID_METADATA', status: 422 }); + expect(refusal!.message).toContain(`Page "sales_home_page" of package "${HOTCRM_ID}": \`isDefault: true\` is refused on a \`type: 'home'\` page`); + expect(ql.registry.getPackage(HOTCRM_ID)).toBeUndefined(); + expect(await packages.get(HOTCRM_ID)).toBeNull(); + expect(lines.filter((l) => l.text.includes('ADR-0087 conversion'))).toEqual([]); + }); +}); + describe('sys_packages boot hydration converts a stored body forward before the registry judges it', () => { it('hydrates HotCRM 4.0.1\'s stored body, converted, with the engine\'s named warning', async () => { const { ql, lines, packages } = await restartWith([hotcrmBody()]); diff --git a/packages/services/service-package/vitest.config.ts b/packages/services/service-package/vitest.config.ts index fbafaeec329..396849ec16d 100644 --- a/packages/services/service-package/vitest.config.ts +++ b/packages/services/service-package/vitest.config.ts @@ -18,7 +18,7 @@ export default defineConfig({ disableConsoleIntercept: true, }, resolve: { - // `hydration-convert-forward.test.ts` boots the REAL engine over a REAL + // `hydration-convert-forward.test.ts` boots the REAL engine, protocol and // driver, so its verdict must be about the source in this checkout, never // a sibling's `dist/` (`pnpm check:test-source-alias`). ANCHORED regex, // array form: a bare string `find` matches by PREFIX and would swallow a @@ -26,6 +26,7 @@ export default defineConfig({ alias: [ { find: /^@objectstack\/core$/, replacement: path.resolve(__dirname, '../../core/src/index.ts') }, { find: /^@objectstack\/objectql$/, replacement: path.resolve(__dirname, '../../objectql/src/index.ts') }, + { find: /^@objectstack\/metadata-protocol$/, replacement: path.resolve(__dirname, '../../metadata-protocol/src/index.ts') }, { find: /^@objectstack\/driver-sql$/, replacement: path.resolve(__dirname, '../../drivers/driver-sql/src/index.ts') }, ], }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5a641003bd5..4c4654cdba9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2827,6 +2827,9 @@ importers: '@objectstack/driver-sql': specifier: workspace:* version: link:../../drivers/driver-sql + '@objectstack/metadata-protocol': + specifier: workspace:* + version: link:../../metadata-protocol '@objectstack/objectql': specifier: workspace:* version: link:../../objectql diff --git a/scripts/cross-package-test-inputs.mjs b/scripts/cross-package-test-inputs.mjs index c991ca96325..8330050d024 100644 --- a/scripts/cross-package-test-inputs.mjs +++ b/scripts/cross-package-test-inputs.mjs @@ -907,6 +907,12 @@ export const CROSS_PACKAGE_TEST_INPUTS = { }, }, '@objectstack/cloud-connection': { + // src/marketplace-install-local-convert-forward.test.ts reads HotCRM 4.0.1's + // published page shape from `@objectstack/objectql`'s fixture -- the ONE + // copy of that artifact, which the registration door's own pin reads too -- + // to drive the marketplace install's ADR-0087 D2 conversion over the real + // engine. A change to the fixture changes what this suite measures. + // // src/canonical-expression-envelopes.test.ts (#12267) imports `maskComments` // from `js-comment-mask.mjs` to decide which text in this package's `src/` is // a comment and which is a `Page` declaration — the same conversion #9367 @@ -917,6 +923,7 @@ export const CROSS_PACKAGE_TEST_INPUTS = { // verdict is a function of it too — the reason the `@objectstack/cli` entry // above declares the pair rather than the module alone. globs: [ + 'packages/objectql/src/__fixtures__/hotcrm-4.0.1-published-pages.artifact.json', 'scripts/js-comment-mask.mjs', 'scripts/js-comment-mask.d.mts', ], @@ -1309,6 +1316,13 @@ export const CROSS_PACKAGE_TEST_INPUTS = { 'docs/**': ['packages/rest/src/meta-state-route-doc-spelling.test.ts'], }, }, + '@objectstack/service-package': { + // src/hydration-convert-forward.test.ts reads HotCRM 4.0.1's published page + // shape from `@objectstack/objectql`'s fixture -- the ONE copy of that + // artifact -- to drive the `sys_packages` boot hydration's ADR-0087 D2 + // conversion over the real engine and driver. + globs: ['packages/objectql/src/__fixtures__/hotcrm-4.0.1-published-pages.artifact.json'], + }, '@objectstack/service-automation': { // src/connector-pull.integration.test.ts (#20919) drives the connector sync // executor through the REAL `rest` provider, imported from diff --git a/turbo.json b/turbo.json index 7c60860de7a..242098a5577 100644 --- a/turbo.json +++ b/turbo.json @@ -325,6 +325,7 @@ "!dist/**", "!coverage/**", "!.turbo/**", + "$TURBO_ROOT$/packages/objectql/src/__fixtures__/hotcrm-4.0.1-published-pages.artifact.json", "$TURBO_ROOT$/scripts/js-comment-mask.mjs", "$TURBO_ROOT$/scripts/js-comment-mask.d.mts" ] @@ -577,6 +578,18 @@ "$TURBO_ROOT$/skills/objectstack-formula/**" ] }, + "@objectstack/service-package#test": { + "dependsOn": ["^build"], + "outputs": [], + "env": ["OS_TEST_TIERS", "OS_TEST_SHARD"], + "inputs": [ + "$TURBO_DEFAULT$", + "!dist/**", + "!coverage/**", + "!.turbo/**", + "$TURBO_ROOT$/packages/objectql/src/__fixtures__/hotcrm-4.0.1-published-pages.artifact.json" + ] + }, "@objectstack/service-automation#test": { "dependsOn": ["^build"], "outputs": [], From fd2b62115ec8c9092f23299530b2967b2c3db2f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 18:48:14 +0000 Subject: [PATCH 5/8] chore(changeset): service-package and cloud-connection convert stored package bodies forward Claude-Session: https://claude.ai/code/session_013JuCowY7RrGXYbGEv3cWsQ Co-authored-by: Claude --- .../23974-package-doors-convert-forward.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .changeset/23974-package-doors-convert-forward.md diff --git a/.changeset/23974-package-doors-convert-forward.md b/.changeset/23974-package-doors-convert-forward.md new file mode 100644 index 00000000000..321f423914f --- /dev/null +++ b/.changeset/23974-package-doors-convert-forward.md @@ -0,0 +1,18 @@ +--- +'@objectstack/service-package': minor +'@objectstack/cloud-connection': minor +--- + +Clause-②: yes (widening) + +fix(service-package,cloud-connection): a stored or installed package body converts forward under ADR-0087 D2 at the `sys_packages` boot hydration and at the marketplace install, and one refused `sys_packages` record no longer stops the hydration of the rest + +The registration door (`ObjectQL.convertPackageBodyForward`, `@objectstack/objectql`) already replays the versioned forward conversion for a package body that declares its `engines.protocol` range. Two more doors that load a body at rest now go through that same entry point, before the registry judges the body: + +- **`sys_packages` boot hydration** (`PackageServicePlugin.start`). A stored record whose range admits this runtime, built before a retirement (for example a `type: 'home'` page with `isDefault: true`), now hydrates converted, with the engine's named warning, where the registry used to refuse it. The stored row is not rewritten, so it converts again at every boot. + - A record the registry still refuses is reported **by name** at `error`, with the refusal attached, and the records after it keep hydrating. Before, the first refusal threw out of the whole loop to a `debug` line, and every durable package after it was absent from the registry with nothing saying so. + - `Hydrated N package(s) from sys_packages into registry` counts the records the registry accepted in that pass. + - The ADR-0087 D1 handshake still runs first: a record whose declared range excludes this runtime's major is refused by it, named, as before. +- **The marketplace install** (`MarketplaceInstallLocalPlugin`, install and boot rehydrate). The body is converted once per load, and that one converted body is what `manifest.register`, the handler binder, the side effects (translations, seed datasets) and the rehydrate's sample-data heal read. Before, `register` converted its own copy while the side readers read the raw ledger entry. The ledger keeps the body as installed. + +Nothing is narrowed. A body with no declared range (a manifest built in code) meets the registry's refusals exactly as before at both doors. The install primitive `protocol.installPackage` (`POST /api/v1/packages`) is authoring-side and stays strict: the registry judges the manifest as handed. From f1443d9bca310525ec1b6add5d6ffd28791aae2a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 19:34:03 +0000 Subject: [PATCH 6/8] fix(scripts): the name-fold loader gate does not count a full path to another package's fixture Its glob-shape rule read packages/objectql/src/__fixtures__/hotcrm-4.0.1-published-pages.artifact.json, declared in scripts/cross-package-test-inputs.mjs, as a reference to the metadata artifact. Only that fully spelled path is set aside; a bare glob on the same line, a climbing path, a wildcard package path and the metadata fixtures directory still count, each pinned in --self-test. Claude-Session: https://claude.ai/code/session_013JuCowY7RrGXYbGEv3cWsQ Co-authored-by: Claude --- scripts/check-position-name-fold-loaders.mjs | 59 +++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/scripts/check-position-name-fold-loaders.mjs b/scripts/check-position-name-fold-loaders.mjs index 7c35bcbfd3e..76c6306bd10 100644 --- a/scripts/check-position-name-fold-loaders.mjs +++ b/scripts/check-position-name-fold-loaders.mjs @@ -146,10 +146,30 @@ export function classifyReference(relPath) { * long file that happens to mention `__fixtures__` in one place and `hotcrm` in * another, and a gate whose red is a coincidence gets weakened rather than * obeyed. + * + * Rule 3 does not count a fully spelled path to ANOTHER package's fixtures + * directory ({@link OTHER_PACKAGE_FIXTURE_PATH}): `packages/objectql/src/ + * __fixtures__/hotcrm-4.0.1-published-pages.artifact.json` names one file in + * one other directory and cannot sweep up this artifact, while matching the + * glob shape would red on it. Only that path is set aside, and the rest of the + * line is still judged, so a bare glob beside it still counts. A path that + * climbs (`..`) or carries a glob character is never set aside — either can + * reach this directory — and this package's own directory is rule 2's, before + * rule 3 is asked at all. */ +const OTHER_PACKAGE_FIXTURE_PATH = /packages\/(?:[\w.-]+\/)+src\/__fixtures__\/[\w.-]+(?:\/[\w.-]+)*/g; + +function withoutOtherPackageFixturePaths(line) { + return line.replace(OTHER_PACKAGE_FIXTURE_PATH, (path) => ( + path.startsWith(`${FIXTURES_DIR}/`) || path.split('/').includes('..') ? path : '' + )); +} + export function referencesArtifact(text) { if (text.includes(ARTIFACT_BASENAME) || text.includes(FIXTURES_DIR)) return true; - for (const line of text.split('\n')) { + for (const raw of text.split('\n')) { + if (!raw.includes('__fixtures__')) continue; + const line = withoutOtherPackageFixturePaths(raw); if (!line.includes('__fixtures__')) continue; if (line.includes('hotcrm') || line.includes('.artifact')) return true; } @@ -316,7 +336,7 @@ let selfTestReachedVerdict = false; const SELF_TEST_BATTERIES = Object.freeze({ 'REVERSE CONTROL. A zero from a scan that cannot report anything is not a': 3, 'The allowed dispositions, each for its own stated reason.': 5, - 'The text matcher, both directions. A matcher that never matches turns': 5, + 'The text matcher, both directions. A matcher that never matches turns': 12, 'The permission-set extractor, on the real artifact shape.': 2, }); @@ -399,6 +419,41 @@ function selfTest() { referencesArtifact("import x from './__fixtures__/widget.json';\n// unrelated: hotcrm#788\n"), false, ); + check( + "matcher does NOT count a full path to another package's fixture", + referencesArtifact(" 'packages/objectql/src/__fixtures__/hotcrm-4.0.1-published-pages.artifact.json',"), + false, + ); + check( + "matcher does NOT count a full path to a nested package's fixture", + referencesArtifact("readFileSync('packages/services/service-x/src/__fixtures__/hotcrm-1.artifact.json')"), + false, + ); + check( + 'matcher still sees a bare fixtures glob', + referencesArtifact("globs: ['__fixtures__/*.artifact.json'],"), + true, + ); + check( + 'matcher still sees a path into this package\'s own fixtures directory', + referencesArtifact("readFileSync('packages/metadata/src/__fixtures__/x.json')"), + true, + ); + check( + 'matcher still sees a bare glob beside another package\'s fixture path on one line', + referencesArtifact("['packages/objectql/src/__fixtures__/a.json', '**/__fixtures__/*.artifact.json']"), + true, + ); + check( + 'matcher still sees another package\'s path that climbs back toward this directory', + referencesArtifact("glob('packages/objectql/src/__fixtures__/../../../metadata/src/__fixtures__/*.artifact.json')"), + true, + ); + check( + 'matcher still sees a wildcard package path', + referencesArtifact("glob('packages/*/src/__fixtures__/*.artifact.json')"), + true, + ); // ── The permission-set extractor, on the real artifact shape. battery('The permission-set extractor, on the real artifact shape.'); From cbec0eac83ea789c58bc39c88270092874ec18c1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 20:37:56 +0000 Subject: [PATCH 7/8] fix(scripts): judge whole, normalized path tokens in the name-fold gate's carve-out Corrects f1443d9bc, whose carve-out let two spellings through and whose docblock said otherwise: a path into packages/metadata/src/__fixtures__/ spelled with a '.' segment was set aside because the raw match was compared, and a match that ended at a glob character or template interpolation was deleted as its glob-free prefix. Rule 3 now sets aside only a whole path token (delimited by a quote, backtick, whitespace, comma, bracket, parenthesis or end of line) that has no '..' segment, is outside the metadata fixtures directory after path.posix.normalize, and is one literal file path with no glob character or '${'. The four spellings the contract review named are self-test cases, with a '.'-segment literal into the guarded directory and a '.'-segment path to another package; the matcher battery floor is 20. Claude-Session: https://claude.ai/code/session_013JuCowY7RrGXYbGEv3cWsQ Co-authored-by: Claude --- scripts/check-position-name-fold-loaders.mjs | 88 ++++++++++++++++---- 1 file changed, 74 insertions(+), 14 deletions(-) diff --git a/scripts/check-position-name-fold-loaders.mjs b/scripts/check-position-name-fold-loaders.mjs index 76c6306bd10..bd90f10a6cb 100644 --- a/scripts/check-position-name-fold-loaders.mjs +++ b/scripts/check-position-name-fold-loaders.mjs @@ -70,7 +70,7 @@ */ import { readFileSync, readdirSync, statSync } from 'node:fs'; -import { join, relative, sep } from 'node:path'; +import { join, posix, relative, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { isEntrypoint } from './invoked-as.mjs'; @@ -148,21 +148,41 @@ export function classifyReference(relPath) { * obeyed. * * Rule 3 does not count a fully spelled path to ANOTHER package's fixtures - * directory ({@link OTHER_PACKAGE_FIXTURE_PATH}): `packages/objectql/src/ - * __fixtures__/hotcrm-4.0.1-published-pages.artifact.json` names one file in - * one other directory and cannot sweep up this artifact, while matching the - * glob shape would red on it. Only that path is set aside, and the rest of the - * line is still judged, so a bare glob beside it still counts. A path that - * climbs (`..`) or carries a glob character is never set aside — either can - * reach this directory — and this package's own directory is rule 2's, before - * rule 3 is asked at all. + * directory: `packages/objectql/src/__fixtures__/hotcrm-4.0.1-published-pages.artifact.json` + * names one file in one other directory and cannot reach this artifact, while + * the glob shape would red on it. The unit judged is a whole PATH TOKEN + * ({@link PATH_TOKEN}): it starts with `packages/` at the start of the line or + * after a quote, backtick, whitespace, `,`, `[` or `(`, and it runs to the + * next quote, backtick, whitespace, `,`, `]`, `)` or the end of the line, so a + * glob character or `${` inside it stays part of the token. A token is set + * aside only when {@link isOtherPackageFixturePath} says so, which needs all + * three: + * + * - no `..` segment; + * - after `path.posix.normalize` (so `packages/./metadata/…` and + * `packages/metadata/./src/…` are read as the directories they are), it + * does not lie in this package's fixtures directory; + * - the normalized token is one literal file path, `packages//src/ + * __fixtures__/` with only word characters, `.` and `-` in its + * segments — so a token carrying `*`, `?`, `[`, `{` or `${` is never set + * aside. + * + * Only such a token leaves the line; the rest of the line is still judged, so + * a bare glob beside it still counts. This package's own directory, spelled + * literally, is rule 2's, before rule 3 is asked at all. */ -const OTHER_PACKAGE_FIXTURE_PATH = /packages\/(?:[\w.-]+\/)+src\/__fixtures__\/[\w.-]+(?:\/[\w.-]+)*/g; +const PATH_TOKEN = /(?<=^|[\s'"`,[(])packages\/[^\s'"`,\])]*/g; +const LITERAL_OTHER_FIXTURE_FILE = /^packages\/(?:[\w.-]+\/)+src\/__fixtures__\/[\w.-]+(?:\/[\w.-]+)*$/; + +export function isOtherPackageFixturePath(token) { + if (token.split('/').includes('..')) return false; + const normalized = posix.normalize(token); + if (normalized === FIXTURES_DIR || normalized.startsWith(`${FIXTURES_DIR}/`)) return false; + return LITERAL_OTHER_FIXTURE_FILE.test(normalized); +} function withoutOtherPackageFixturePaths(line) { - return line.replace(OTHER_PACKAGE_FIXTURE_PATH, (path) => ( - path.startsWith(`${FIXTURES_DIR}/`) || path.split('/').includes('..') ? path : '' - )); + return line.replace(PATH_TOKEN, (token) => (isOtherPackageFixturePath(token) ? '' : token)); } export function referencesArtifact(text) { @@ -336,7 +356,7 @@ let selfTestReachedVerdict = false; const SELF_TEST_BATTERIES = Object.freeze({ 'REVERSE CONTROL. A zero from a scan that cannot report anything is not a': 3, 'The allowed dispositions, each for its own stated reason.': 5, - 'The text matcher, both directions. A matcher that never matches turns': 12, + 'The text matcher, both directions. A matcher that never matches turns': 20, 'The permission-set extractor, on the real artifact shape.': 2, }); @@ -454,6 +474,46 @@ function selfTest() { referencesArtifact("glob('packages/*/src/__fixtures__/*.artifact.json')"), true, ); + check( + 'matcher still sees a `.`-segment path to this directory, in a template literal', + referencesArtifact("readFileSync(`packages/./metadata/src/__fixtures__/hotcrm-17.1-${kind}.artifact.json`)"), + true, + ); + check( + 'matcher still sees a `.`-segment glob into this directory (package segment)', + referencesArtifact("glob('packages/./metadata/src/__fixtures__/hotcrm-*.artifact.json')"), + true, + ); + check( + 'matcher still sees a `.`-segment glob into this directory (src segment)', + referencesArtifact("glob('packages/metadata/./src/__fixtures__/hotcrm-*.artifact.json')"), + true, + ); + check( + 'matcher still sees another package\'s fixtures path that a glob continues', + referencesArtifact("glob('packages/objectql/src/__fixtures__/a*.artifact.json')"), + true, + ); + check( + 'matcher still sees another package\'s fixtures path that a template continues', + referencesArtifact("readFileSync(`packages/objectql/src/__fixtures__/${name}.artifact.json`)"), + true, + ); + check( + 'matcher still sees a fixtures path that does not START a token', + referencesArtifact("readFileSync('vendor/packages/objectql/src/__fixtures__/hotcrm.artifact.json')"), + true, + ); + check( + 'matcher still sees a `.`-segment LITERAL path into this directory (only normalization reads it)', + referencesArtifact("readFileSync('packages/./metadata/src/__fixtures__/hotcrm-17.1-built-bare-root-predicates.artifact.json')"), + true, + ); + check( + "matcher does NOT count a `.`-segment literal path to another package's fixture", + referencesArtifact(" 'packages/./objectql/src/__fixtures__/hotcrm-4.0.1-published-pages.artifact.json',"), + false, + ); // ── The permission-set extractor, on the real artifact shape. battery('The permission-set extractor, on the real artifact shape.'); From bb3838c8153b74cf54ebdb33c1b0665376851190 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 22:27:27 +0000 Subject: [PATCH 8/8] test(service-package,cloud-connection): the converted bodies keep the origin main's registry rows record After merging main, the registry row carries the caller's origin. The marketplace pin's manifest wrapper now forwards register's options, and both door pins assert that the converted body still lands as 'authored' (sys_packages) or 'marketplace' (install, rehydrate). Claude-Session: https://claude.ai/code/session_013JuCowY7RrGXYbGEv3cWsQ Co-authored-by: Claude --- .../marketplace-install-local-convert-forward.test.ts | 11 ++++++----- .../src/hydration-convert-forward.test.ts | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/cloud-connection/src/marketplace-install-local-convert-forward.test.ts b/packages/cloud-connection/src/marketplace-install-local-convert-forward.test.ts index 16e2b066ae9..1734ec4a30f 100644 --- a/packages/cloud-connection/src/marketplace-install-local-convert-forward.test.ts +++ b/packages/cloud-connection/src/marketplace-install-local-convert-forward.test.ts @@ -103,13 +103,13 @@ async function bootPlugin() { }; const handed: Array<{ reader: string; body: Bag }> = []; - const realManifest = kernel.getService<{ register(m: unknown): unknown }>('manifest'); + const realManifest = kernel.getService<{ register(m: unknown, options?: unknown): unknown }>('manifest'); const manifest = { - register: async (body: Bag) => { + register: async (body: Bag, options?: unknown) => { // The plugin also registers its own Setup UI bundle at // `kernel:ready`; only the package under test is recorded. if (body?.id === HOTCRM_ID) handed.push({ reader: 'register', body }); - await realManifest.register(body); + await realManifest.register(body, options); }, }; const routes = new Map(); @@ -184,7 +184,8 @@ describe('the marketplace install converts a body once and every reader of the l const { ql, handed, conversionWarnings } = await bootPlugin(); expectOneConvertedBody(handed, ['register', 'bindArtifactHandlers', 'applySideEffects', 'heal']); - expect(ql.registry.getPackage(HOTCRM_ID)).toBeDefined(); + // The converted body still registers as a marketplace install. + expect(ql.registry.getPackage(HOTCRM_ID)?.source).toBe('marketplace'); // Converted once: `register` met the entry point again with the // converted body, and no second warning was printed. expect(conversionWarnings.filter((w) => w.includes(`conversion '${IS_DEFAULT_CONVERSION}'`))).toHaveLength(1); @@ -199,7 +200,7 @@ describe('the marketplace install converts a body once and every reader of the l expect(res.status, JSON.stringify(res.payload)).toBe(200); expectOneConvertedBody(handed, ['register', 'bindArtifactHandlers', 'applySideEffects']); - expect(ql.registry.getPackage(HOTCRM_ID)).toBeDefined(); + expect(ql.registry.getPackage(HOTCRM_ID)?.source).toBe('marketplace'); expect(conversionWarnings.filter((w) => w.includes(`conversion '${IS_DEFAULT_CONVERSION}'`))).toHaveLength(1); expect(homePage(new LocalManifestSource(dir).read(HOTCRM_ID).entry!.manifest as Bag)).toMatchObject({ isDefault: true }); }); diff --git a/packages/services/service-package/src/hydration-convert-forward.test.ts b/packages/services/service-package/src/hydration-convert-forward.test.ts index 8070ce61833..938065bb67c 100644 --- a/packages/services/service-package/src/hydration-convert-forward.test.ts +++ b/packages/services/service-package/src/hydration-convert-forward.test.ts @@ -204,6 +204,8 @@ describe('sys_packages boot hydration converts a stored body forward before the const record = ql.registry.getPackage(HOTCRM_ID); expect(record, 'the stored HotCRM body was not hydrated').toBeDefined(); + // The converted body is still recorded as the authored package it was installed as. + expect(record!.source).toBe('authored'); const home = ((record!.manifest as Bag).pages as Bag[]).find((p) => p.name === 'sales_home_page')!; expect(home).toMatchObject({ type: 'home' }); expect(home).not.toHaveProperty('isDefault');