From 44f958b37baa43d45fc626fe6f4e961a1b37f414 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 22:22:44 +0000 Subject: [PATCH 1/2] fix(metadata-protocol): a failed seed publish counts the rows earlier seeds wrote The seed loader records, on the error a load throws, the rows it had written (finished datasets plus the dataset that threw) and the object of the dataset that threw. applySeedBodies' failure receipt reads it, so seedApplied.inserted/updated describe what landed and error names the seed that stopped the load. Publishing and the stop-at-failure behaviour are unchanged. Claude-Session: https://claude.ai/code/session_01L6bhgUCizoYWQVJWjhdR6F Co-authored-by: Claude --- .../27393-seed-receipt-partial-counts.md | 18 ++ .../src/protocol-publish-batch.ts | 25 +- .../src/seed-loader-partial-writes.test.ts | 148 ++++++++++ packages/metadata-protocol/src/seed-loader.ts | 93 ++++++- ...package-drafts-seed-receipt-counts.test.ts | 260 ++++++++++++++++++ 5 files changed, 528 insertions(+), 16 deletions(-) create mode 100644 .changeset/27393-seed-receipt-partial-counts.md create mode 100644 packages/metadata-protocol/src/seed-loader-partial-writes.test.ts create mode 100644 packages/objectql/src/publish-package-drafts-seed-receipt-counts.test.ts diff --git a/.changeset/27393-seed-receipt-partial-counts.md b/.changeset/27393-seed-receipt-partial-counts.md new file mode 100644 index 00000000000..06c180cddd4 --- /dev/null +++ b/.changeset/27393-seed-receipt-partial-counts.md @@ -0,0 +1,18 @@ +--- +'@objectstack/metadata-protocol': patch +--- + +fix(metadata-protocol): when a seed in a package publish fails, the `seedApplied` receipt counts the rows the earlier seeds wrote and names the seed that stopped the load + +Clause-②: no +Metadata-protocol: none + +`POST /api/v1/packages/:id/publish-drafts` applies every published seed body in one load. The load writes seed by seed and stops at the first one that throws (for example, a seed naming an object that does not exist); the rows the seeds ahead of it wrote stay written. The receipt used to answer `inserted: 0, updated: 0` all the same, so an author or Studio reading it was told nothing had been written while the table held those rows. + +Now the receipt keeps its shape and its counters describe what was written: + +- `inserted` / `updated` are the rows the load wrote before it stopped, including any rows the failing seed itself wrote before it threw. +- `success` is `false`, as before. +- `error` names the seed that stopped the load by the object its dataset populates: `seed apply stopped at the seed for object 'OBJECT': ...`, followed by the same sentence as before (a refusal the error declared is quoted, anything else is withheld as `seed apply failed`). + +Unchanged: which seeds are published, that the seeds after the failing one are not applied, and that nothing is rolled back. A throw before the loader writes anything (a malformed seed body, for example) still answers `inserted: 0, updated: 0` and the same `error` as before. diff --git a/packages/metadata-protocol/src/protocol-publish-batch.ts b/packages/metadata-protocol/src/protocol-publish-batch.ts index 75bf877afdb..838c1b9a1d2 100644 --- a/packages/metadata-protocol/src/protocol-publish-batch.ts +++ b/packages/metadata-protocol/src/protocol-publish-batch.ts @@ -11,6 +11,8 @@ * `ObjectStackProtocolImplementation` method of the same name as it stood, * re-indented one level and otherwise unedited: the same reads, in the same * order, answering with the same receipts and the same degradations. + * Edited since the move: `applySeedBodies`' failure receipt counts the rows the + * loader wrote before it threw and names the seed that stopped it. * * ## How the moved bodies reach the class * @@ -46,6 +48,7 @@ import { seedRequestValidationError, } from './protocol.js'; import { CLOSURE_CONTEXT_KEY_BY_TYPE, type RuntimePendingDeclarations } from './runtime-authoring-gate.js'; +import type { SeedLoadPartialWrites } from './seed-loader.js'; /** * The `ObjectStackProtocolImplementation` members the functions below read — @@ -72,6 +75,9 @@ export async function applySeedBodies( success: boolean; inserted: number; updated: number; error?: string; errors?: unknown[]; issues?: Array<{ path: string; message: string; code?: string | undefined }>; }> { + // The loader's reader of what a THROWN load had written. Unset until the + // loader module is imported, which no write precedes. + let partialWritesOf: ((e: unknown) => SeedLoadPartialWrites | undefined) | undefined; try { const seeds = bodies.filter( (b: any) => b && typeof b.object === 'string' && Array.isArray(b.records), @@ -79,7 +85,8 @@ export async function applySeedBodies( if (seeds.length === 0) { return { success: false, inserted: 0, updated: 0, error: 'seed apply: no readable seed bodies' }; } - const { SeedLoaderService } = await import('./seed-loader.js'); + const { SeedLoaderService, seedLoadPartialWrites } = await import('./seed-loader.js'); + partialWritesOf = seedLoadPartialWrites; const { SeedLoaderRequestSchema } = await import('@objectstack/spec/data'); // The loader only needs `getObject` from IMetadataService (dependency // graph + field introspection); satisfy it from the protocol's own @@ -134,9 +141,21 @@ export async function applySeedBodies( // which key. That declaration is the whole reason this site could // be converted rather than escalated. console.warn(`[protocol.applySeedBodies] seed apply failed: ${e?.message ?? e}`); + // The counters describe what was WRITTEN. The loader writes dataset + // by dataset and stops at the one that throws; the rows of the + // datasets ahead of it stay written (nothing rolls them back), so the + // receipt carries the loader's own count of them and names the seed + // that stopped it. No reading means the throw came before the loader + // wrote anything — the request parse, or the loader's import. + const written = partialWritesOf?.(e); + const failure = clientFacingFailureText(e, 'seed apply failed'); return { - success: false, inserted: 0, updated: 0, - error: clientFacingFailureText(e, 'seed apply failed'), + success: false, + inserted: written?.inserted ?? 0, + updated: written?.updated ?? 0, + error: written?.failedObject + ? `seed apply stopped at the seed for object '${written.failedObject}': ${failure}` + : failure, // [#10524] A DECLARED refusal's structured findings ride the // receipt beside the headline `error` — the message is a // one-sentence headline now, so this is where the per-key diff --git a/packages/metadata-protocol/src/seed-loader-partial-writes.test.ts b/packages/metadata-protocol/src/seed-loader-partial-writes.test.ts new file mode 100644 index 00000000000..66f7818591b --- /dev/null +++ b/packages/metadata-protocol/src/seed-loader-partial-writes.test.ts @@ -0,0 +1,148 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `seedLoadPartialWrites` — what a load that THREW had already written. + * + * A load writes dataset by dataset and rolls nothing back when a later one + * throws, so the caller reporting the failure (the publish batch's + * `seedApplied` receipt) needs the rows that landed: the finished datasets' + * and those of the dataset that threw part-way. The publish-door pins live in + * `@objectstack/objectql`'s `publish-package-drafts-seed-receipt-counts.test.ts`; + * this file pins the one case that door cannot stage — a throw INSIDE a + * dataset after some of its rows were written (here: the file service failing + * while a written row's seed file is settled) — and the throw that comes + * before any dataset. + * + * Every count is checked against the fake engine's store. + */ + +import { createHash } from 'node:crypto'; +import { describe, it, expect, vi } from 'vitest'; +import type { IDataEngine, IFieldFileService, IMetadataService } from '@objectstack/spec/contracts'; +import { seedFile } from '@objectstack/spec/data'; +import { assertEngineDeleteDispatch, assertEngineUpdateDispatch, assertEngineFindOnePredicate } from '@objectstack/metadata-core'; +import { SeedLoaderService, seedLoadPartialWrites } from './seed-loader.js'; + +function createLogger() { + return { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }; +} + +function createEngine(): { engine: IDataEngine; store: Record } { + const store: Record = {}; + let next = 0; + const matches = (row: Record, where: Record) => + Object.entries(where).every(([k, v]) => { + if (k.startsWith('$')) throw new Error(`fake driver: unsupported operator ${k}`); + return row[k] === v; + }); + const engine = { + find: vi.fn(async (object: string, query?: any) => { + let rows = store[object] ?? []; + if (query?.where) rows = rows.filter((r) => matches(r, query.where)); + return rows.map((r) => ({ ...r })); + }), + findOne: vi.fn(async (object: string, query?: any) => { + assertEngineFindOnePredicate(object, query); + const rows = (store[object] ?? []).filter((r) => matches(r, query?.where ?? {})); + return rows[0] ? { ...rows[0] } : null; + }), + insert: vi.fn(async (object: string, data: any) => { + store[object] ??= []; + const one = (d: any) => { + const row = { id: `row-${++next}`, ...d }; + store[object].push(row); + return { ...row }; + }; + return Array.isArray(data) ? data.map(one) : one(data); + }), + update: vi.fn(async (object: string, data: any) => { + assertEngineUpdateDispatch(data, undefined); + const rows = store[object] ?? []; + const i = rows.findIndex((r) => r.id === data.id); + if (i >= 0) rows[i] = { ...rows[i], ...data }; + return data; + }), + delete: vi.fn(async (_object: string, options?: any) => { + assertEngineDeleteDispatch(options); + return { deleted: 1 }; + }), + count: vi.fn(async (object: string) => (store[object] ?? []).length), + aggregate: vi.fn(async () => []), + } as unknown as IDataEngine; + return { engine, store }; +} + +/** A `field-files` service that stores files and is DOWN for every read of them. */ +function createFieldFilesDownOnRead(): IFieldFileService { + let next = 0; + return { + storeFile: vi.fn(async (input: any) => ({ fileId: `file-${++next}`, size: input.data.byteLength })), + describeFiles: vi.fn(async () => { throw new Error('the storage service is down'); }), + claimFiles: vi.fn(async () => {}), + } as unknown as IFieldFileService; +} + +/** `pw_tag` (plain) is loaded first: `pw_doc` looks it up. */ +function createMetadata(fail?: Error): IMetadataService { + const objects: Record = { + pw_tag: { name: 'pw_tag', fields: { name: { type: 'text' } } }, + pw_doc: { + name: 'pw_doc', + fields: { code: { type: 'text' }, file: { type: 'file' }, tag: { type: 'lookup', reference: 'pw_tag' } }, + }, + }; + return { + getObject: vi.fn(async (name: string) => { if (fail) throw fail; return objects[name]; }), + } as unknown as IMetadataService; +} + +const CONFIG = { dryRun: false, haltOnError: false, multiPass: true, defaultMode: 'upsert', batchSize: 1000, transaction: false } as any; + +const pdf = Buffer.from('%PDF pw'); +const FILES = { + 'fixtures/a.pdf': { + contentType: 'application/pdf', + size: pdf.byteLength, + sha256: createHash('sha256').update(pdf).digest('hex'), + data: pdf.toString('base64'), + }, +}; + +const seeds = () => [ + { object: 'pw_doc', externalId: 'code', mode: 'upsert', env: ['prod', 'dev', 'test'], files: FILES, + records: [{ code: 'D1', tag: 'T1', file: seedFile('fixtures/a.pdf') }] }, + { object: 'pw_tag', mode: 'upsert', env: ['prod', 'dev', 'test'], records: [{ name: 'T1' }, { name: 'T2' }] }, +] as any; + +describe('seedLoadPartialWrites', () => { + it('counts the finished datasets AND the rows the throwing dataset wrote, naming its object', async () => { + const { engine, store } = createEngine(); + const loader = new SeedLoaderService(engine, createMetadata(), createLogger() as any, { + getFieldFiles: () => createFieldFilesDownOnRead(), + }); + + const err = await loader.load({ seeds: seeds(), config: CONFIG }).then(() => undefined, (e: unknown) => e); + + expect((err as Error).message).toBe('the storage service is down'); + // At the engine: both tags, and the doc row written before its file settle threw. + expect((store.pw_tag ?? []).map((r) => r.name).sort()).toEqual(['T1', 'T2']); + expect((store.pw_doc ?? []).map((r) => r.code)).toEqual(['D1']); + expect(seedLoadPartialWrites(err)).toEqual({ inserted: 3, updated: 0, failedObject: 'pw_doc' }); + }); + + it('a load that throws before any dataset wrote: zero, and no object named', async () => { + const { engine, store } = createEngine(); + const loader = new SeedLoaderService(engine, createMetadata(new Error('metadata store unavailable')), createLogger() as any); + + const err = await loader.load({ seeds: seeds(), config: CONFIG }).then(() => undefined, (e: unknown) => e); + + expect((err as Error).message).toBe('metadata store unavailable'); + expect(store).toEqual({}); + expect(seedLoadPartialWrites(err)).toEqual({ inserted: 0, updated: 0 }); + }); + + it('an error that did not come out of a load carries no reading', () => { + expect(seedLoadPartialWrites(new Error('elsewhere'))).toBeUndefined(); + expect(seedLoadPartialWrites('a string')).toBeUndefined(); + }); +}); diff --git a/packages/metadata-protocol/src/seed-loader.ts b/packages/metadata-protocol/src/seed-loader.ts index f49043d82f9..6557a0c4b94 100644 --- a/packages/metadata-protocol/src/seed-loader.ts +++ b/packages/metadata-protocol/src/seed-loader.ts @@ -558,6 +558,35 @@ interface ReplayComparands { readonly masked: ReadonlySet; } +/** + * What a {@link SeedLoaderService.load} that THREW had already written. + * + * A load writes dataset by dataset and nothing undoes a dataset's rows when a + * later one throws, so a caller that reports the failure must still be able to + * say what landed: a receipt answering `inserted: 0` while the table holds the + * earlier datasets' rows is a receipt that lies. Read with + * {@link seedLoadPartialWrites} on the error the load threw. + */ +export interface SeedLoadPartialWrites { + /** Rows inserted before the load threw — finished datasets plus the one that threw. */ + readonly inserted: number; + /** Rows updated before the load threw, counted the same way. */ + readonly updated: number; + /** The object of the dataset that was loading when the load threw; absent when it threw outside any dataset. */ + readonly failedObject?: string; +} + +/** The writes a thrown load had made, keyed by the error it threw. */ +const partialWritesByError = new WeakMap(); + +/** + * What the {@link SeedLoaderService.load} that threw `err` had written, or + * `undefined` when `err` did not come out of a load. + */ +export function seedLoadPartialWrites(err: unknown): SeedLoadPartialWrites | undefined { + return err !== null && typeof err === 'object' ? partialWritesByError.get(err) : undefined; +} + /** What a {@link SeedLoaderService} is built with beyond its engine, metadata and logger. */ export interface SeedLoaderServiceOptions { /** @@ -703,6 +732,17 @@ export class SeedLoaderService implements ISeedLoaderService { */ private uncomparedCredentials = new Map; rows: number }>(); + /** + * The CURRENT {@link load}'s writes so far, for {@link SeedLoadPartialWrites}: + * the finished datasets' results, and the counters of the dataset loading + * now. Reset per `load`; an instance field for {@link summariesStale}'s + * reason (datasets load strictly sequentially). + */ + private loadProgress?: { + finished: readonly SeedLoadResultParsed[]; + loading?: { object: string; written: { inserted: number; updated: number } }; + }; + /** * [#25779] The `field-files` service (`FIELD_FILE_SERVICE`) a seed file is * stored through, read per dataset. Absent — a loader built without one, or @@ -744,10 +784,34 @@ export class SeedLoaderService implements ISeedLoaderService { * it. */ async load(request: SeedLoaderRequestParsed): Promise { - return runWithAdvisoryAggregation( - () => this.loadDatasets(request), - (groups) => this.reportAdvisorySummary(groups), - ); + this.loadProgress = undefined; + try { + return await runWithAdvisoryAggregation( + () => this.loadDatasets(request), + (groups) => this.reportAdvisorySummary(groups), + ); + } catch (err) { + throw this.notePartialWrites(err); + } + } + + /** + * Record on `err` what this load wrote before it threw (see + * {@link SeedLoadPartialWrites}) and hand back the error to rethrow — the + * same error, so its declared status and message reach the caller as they + * did; a thrown non-object is wrapped, since only an object can carry it. + */ + private notePartialWrites(err: unknown): unknown { + const finished = this.loadProgress?.finished ?? []; + const loading = this.loadProgress?.loading; + const partial: SeedLoadPartialWrites = { + inserted: finished.reduce((sum, r) => sum + r.inserted, 0) + (loading?.written.inserted ?? 0), + updated: finished.reduce((sum, r) => sum + r.updated, 0) + (loading?.written.updated ?? 0), + ...(loading ? { failedObject: loading.object } : {}), + }; + const carrier = err !== null && typeof err === 'object' ? err : new Error(String(err)); + partialWritesByError.set(carrier, partial); + return carrier; } /** @@ -852,6 +916,7 @@ export class SeedLoaderService implements ISeedLoaderService { this.warnOnUnresolvedLocaleScope(config, request.seeds); const allErrors: ReferenceResolutionError[] = []; const allResults: SeedLoadResultParsed[] = []; + this.loadProgress = { finished: allResults }; // Per-load counter — a service instance can be reused across loads. this.summariesStale = 0; // [#9071] Same per-load lifetime, same reason: a publish between two loads @@ -1127,8 +1192,9 @@ export class SeedLoaderService implements ISeedLoaderService { // `matchesExistingRow` in the record loop below. const createOnlyColumns = createOnlySeedColumns(dataset, externalId); - let inserted = 0; - let updated = 0; + // Read by `notePartialWrites` should this dataset throw part-way. + const written = { inserted: 0, updated: 0 }; + if (this.loadProgress) this.loadProgress.loading = { object: objectName, written }; let skipped = 0; let errored = 0; let referencesResolved = 0; @@ -1342,7 +1408,7 @@ export class SeedLoaderService implements ISeedLoaderService { for (const res of writeResults) { const { recordIndex, externalIdValue, record } = batch[res.index]; if (res.ok) { - inserted++; + written.inserted++; const internalId = this.extractId(res.record); if (internalId) noteLanded(recordIndex, internalId); // [commit 9a884c6e4] if (externalIdValue && internalId) { @@ -2046,8 +2112,8 @@ export class SeedLoaderService implements ISeedLoaderService { objectName, record, mode, externalId, existingRecords, comparands ); - if (result.action === 'inserted') inserted++; - else if (result.action === 'updated') updated++; + if (result.action === 'inserted') written.inserted++; + else if (result.action === 'updated') written.updated++; else if (result.action === 'skipped') { skipped++; if (result.uncompared) this.noteUncomparedCredentials(objectName, result.uncompared); // [#24624] @@ -2110,7 +2176,7 @@ export class SeedLoaderService implements ISeedLoaderService { } try { await this.writeRecoveringSummary(objectName, () => withTransientRetry(() => this.engine.update(objectName, { ...record, id: decision.id }, opts))); - updated++; + written.updated++; await settleSeedFiles(i, decision.id); // [#25779] } catch (err: any) { errored++; @@ -2142,7 +2208,7 @@ export class SeedLoaderService implements ISeedLoaderService { if (externalIdValue) { insertedRecords.get(objectName)!.set(externalIdValue, `dry-run-id-${i}`); } - inserted++; // Count as "would be inserted" + written.inserted++; // Count as "would be inserted" } } @@ -2176,11 +2242,12 @@ export class SeedLoaderService implements ISeedLoaderService { .map(([, id]) => id), )]; + if (this.loadProgress) this.loadProgress.loading = undefined; return { object: objectName, mode, - inserted, - updated, + inserted: written.inserted, + updated: written.updated, skipped, errored, total: dataset.records.length, diff --git a/packages/objectql/src/publish-package-drafts-seed-receipt-counts.test.ts b/packages/objectql/src/publish-package-drafts-seed-receipt-counts.test.ts new file mode 100644 index 00000000000..60e62236f32 --- /dev/null +++ b/packages/objectql/src/publish-package-drafts-seed-receipt-counts.test.ts @@ -0,0 +1,260 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The `seedApplied` receipt of a package publish counts what was WRITTEN — on + * a failure too. + * + * `publishPackageDrafts` applies every seed body of the batch in one loader + * call. When one seed fails (here: it names an object that does not exist), + * the datasets ahead of it have already written their rows, and nothing undoes + * them. The receipt used to answer `inserted: 0, updated: 0` all the same — a + * receipt saying nothing was written while the table holds the rows. + * + * The semantics this file pins: **truthful aggregate counts**. `inserted` / + * `updated` are the rows the loader wrote before it stopped, `success` is + * `false`, and `error` names the seed (by the object its dataset populates) + * that stopped it. Which seeds are published and the fact that the loader + * stops at the failing dataset are unchanged; nothing is rolled back. + * + * Every count is checked against the ENGINE (`engine.find` on the seeded + * object), never against a value derived from the code under test. + * + * Driven through the real door: the real protocol, a real ObjectQL engine and + * the real `SeedLoaderService`, over an in-memory driver (harness copied from + * `publish-package-drafts-response-conformance.test.ts` — copied, not shared, + * so each file can fail on its own; this package carries no sqlite driver). + */ +import { describe, it, expect } from 'vitest'; +import type { ServiceObject } from '@objectstack/spec/data'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +import { ObjectQL } from './engine.js'; +import { SysMetadataAuditObject, SysMetadataCommitObject, SysMetadataHistoryObject } from '@objectstack/metadata-core'; + +const sysMetadataObject: ServiceObject = { + name: 'sys_metadata', + label: 'System Metadata', + fields: { + id: { name: 'id', label: 'ID', type: 'text' as const }, + type: { name: 'type', label: 'Type', type: 'text' as const, required: true }, + name: { name: 'name', label: 'Name', type: 'text' as const, required: true }, + organization_id: { name: 'organization_id', label: 'Org', type: 'text' as const }, + package_id: { name: 'package_id', label: 'Package', type: 'text' as const }, + metadata: { name: 'metadata', label: 'Body', type: 'textarea' as const }, + draft_changes: { name: 'draft_changes', label: 'Draft Changes', type: 'textarea' as const }, + checksum: { name: 'checksum', label: 'Checksum', type: 'text' as const, maxLength: 71 }, + state: { name: 'state', label: 'State', type: 'text' as const }, + version: { name: 'version', label: 'Version', type: 'number' as const }, + created_at: { name: 'created_at', label: 'Created', type: 'datetime' as const }, + updated_at: { name: 'updated_at', label: 'Updated', type: 'datetime' as const }, + }, +}; + +function makeMemoryDriver() { + const stores = new Map>>(); + const storeFor = (obj: string) => { + let s = stores.get(obj); + if (!s) { s = new Map(); stores.set(obj, s); } + return s; + }; + let nextId = 0; + const matchesWhere = (row: Record, where: any): boolean => { + if (!where || typeof where !== 'object') return true; + for (const [k, v] of Object.entries(where)) { + if (k === '$and' && Array.isArray(v)) { + if (!v.every((w: any) => matchesWhere(row, w))) return false; + continue; + } + if (k === '$or' && Array.isArray(v)) { + if (!v.some((w: any) => matchesWhere(row, w))) return false; + continue; + } + if (k.startsWith('$')) continue; + const rowVal = row[k]; + const expected = (v && typeof v === 'object' && '$eq' in (v as any)) ? (v as any).$eq : v; + const a = rowVal === undefined ? null : rowVal; + const b = expected === undefined ? null : expected; + if (a !== b) return false; + } + return true; + }; + const driver: any = { + name: 'memory', version: '0.0.0', supports: {} as any, + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, + async execute() { return null; }, + async find(object: string, ast: any) { + return Array.from(storeFor(object).values()).filter((r) => matchesWhere(r, ast?.where)); + }, + async findOne(object: string, ast: any) { + for (const r of storeFor(object).values()) if (matchesWhere(r, ast?.where)) return r; + return null; + }, + async create(object: string, data: Record) { + nextId += 1; + const id = (data.id as string) ?? `r_${nextId}`; + const row = { ...data, id }; + storeFor(object).set(id, row); + return row; + }, + async update(object: string, id: string, data: Record) { + const s = storeFor(object); + const cur = s.get(id); + if (!cur) throw new Error(`not found: ${object}/${id}`); + const updated = { ...cur, ...data, id }; + s.set(id, updated); + return updated; + }, + async upsert(object: string, data: Record) { + const id = data.id as string | undefined; + if (id && storeFor(object).has(id)) return this.update(object, id, data); + return this.create(object, data); + }, + async delete(object: string, id: string) { return storeFor(object).delete(id); }, + async count(object: string, ast: any) { return (await this.find(object, ast)).length; }, + async bulkCreate(object: string, rows: Record[]) { + return Promise.all(rows.map((r) => this.create(object, r))); + }, + async bulkUpdate() { return []; }, async bulkDelete() {}, + async beginTransaction() { return { commit: async () => {}, rollback: async () => {} }; }, + async commit() {}, async rollback() {}, + }; + return { driver }; +} + +const PKG = 'com.objectstack.qa.seedrepro'; +const ITEM = 'seedrepro_item'; +const MISSING = 'seedrepro_nope'; + +/** + * The real protocol over a real engine, with the package installed and ITS + * object published through the same door — so the seeded table is one the + * batch itself put there, exactly as the reproduction staged it. + */ +async function makeProtocol() { + const engine = new ObjectQL(); + const { driver } = makeMemoryDriver(); + engine.registerDriver(driver, true); + await engine.init(); + engine.registry.registerObject(sysMetadataObject, 'test-package'); + for (const o of [SysMetadataHistoryObject, SysMetadataAuditObject, SysMetadataCommitObject]) engine.registry.registerObject(o as any, 'test-package'); + engine.registry.installPackage({ id: PKG, name: 'R', version: '1.0.0' } as never, 'code'); + const p = new ObjectStackProtocolImplementation(engine); + await stage(p, [{ type: 'object', name: ITEM, item: { + name: ITEM, label: 'R', sharingModel: 'public_read_write', + fields: { name: { name: 'name', label: 'Name', type: 'text' } }, + } }]); + const objectPublish: any = await p.publishPackageDrafts({ packageId: PKG }); + expect(objectPublish.outcome).toBe('published'); + return { p, engine }; +} + +/** `PUT /meta/:type/:name?mode=draft&package=PKG` for each draft. */ +async function stage( + p: ObjectStackProtocolImplementation, + drafts: Array<{ type: string; name: string; item: unknown }>, +): Promise { + for (const d of drafts) { + await (p as any).saveMetaItem({ type: d.type, name: d.name, item: d.item, mode: 'draft', packageId: PKG }); + } +} + +/** The seeded object's rows, read at the engine. */ +async function rowsAtEngine(engine: ObjectQL): Promise { + const rows = await engine.find(ITEM, { context: { isSystem: true } } as any); + return (rows as Array<{ name?: string }>).map((r) => String(r.name)).sort(); +} + +const goodSeed = (names: string[]) => ({ object: ITEM, mode: 'upsert', records: names.map((name) => ({ name })) }); +const missingObjectSeed = { object: MISSING, mode: 'upsert', records: [{ name: 'Z' }] }; + +describe('publish-drafts seedApplied counts what was written', () => { + it('seed 1 writes 2 rows, seed 2 names a missing object: inserted 2, success false, error names seed 2', async () => { + const { p, engine } = await makeProtocol(); + await stage(p, [ + { type: 'seed', name: 'r_good', item: goodSeed(['R1', 'R2']) }, + { type: 'seed', name: 'r_missing_object', item: missingObjectSeed }, + ]); + + const res: any = await p.publishPackageDrafts({ packageId: PKG }); + + const written = await rowsAtEngine(engine); + expect(written).toEqual(['R1', 'R2']); + expect(res.seedApplied.success).toBe(false); + expect(res.seedApplied.inserted).toBe(written.length); + expect(res.seedApplied.updated).toBe(0); + expect(res.seedApplied.error).toContain(`'${MISSING}'`); + // Unchanged by this fix: both seeds are published. + expect(res.outcome).toBe('published'); + expect(res.publishedCount).toBe(2); + }); + + it('seed 1 UPDATES 2 existing rows, seed 2 fails: updated 2, inserted 0', async () => { + const { p, engine } = await makeProtocol(); + await stage(p, [{ type: 'seed', name: 'r_good', item: goodSeed(['R1', 'R2']) }]); + const first: any = await p.publishPackageDrafts({ packageId: PKG }); + expect(first.seedApplied).toEqual({ success: true, inserted: 2, updated: 0 }); + + // The same natural keys with a changed column: an upsert UPDATE each. + await stage(p, [ + { type: 'seed', name: 'r_good', item: { ...goodSeed([]), records: [ + { name: 'R1', label: 'one' }, { name: 'R2', label: 'two' }, + ] } }, + { type: 'seed', name: 'r_missing_object', item: missingObjectSeed }, + ]); + await (p as any).saveMetaItem({ type: 'object', name: ITEM, mode: 'draft', packageId: PKG, item: { + name: ITEM, label: 'R', sharingModel: 'public_read_write', + fields: { name: { name: 'name', label: 'Name', type: 'text' }, label: { name: 'label', label: 'Label', type: 'text' } }, + } }); + + const res: any = await p.publishPackageDrafts({ packageId: PKG }); + + const rows = await engine.find(ITEM, { context: { isSystem: true } } as any) as Array<{ name: string; label?: string }>; + expect(rows.map((r) => `${r.name}:${r.label}`).sort()).toEqual(['R1:one', 'R2:two']); + expect(res.seedApplied.success).toBe(false); + expect(res.seedApplied.updated).toBe(2); + expect(res.seedApplied.inserted).toBe(0); + expect(res.seedApplied.error).toContain(`'${MISSING}'`); + }); + + it('[control] every seed succeeds: the counts are what they always were', async () => { + const { p, engine } = await makeProtocol(); + await stage(p, [ + { type: 'seed', name: 'r_good', item: goodSeed(['R1', 'R2']) }, + { type: 'seed', name: 'r_more', item: goodSeed(['R3']) }, + ]); + + const res: any = await p.publishPackageDrafts({ packageId: PKG }); + + expect(await rowsAtEngine(engine)).toEqual(['R1', 'R2', 'R3']); + expect(res.seedApplied).toEqual({ success: true, inserted: 3, updated: 0 }); + }); + + it('the FIRST seed fails, a good one follows: nothing was written, and the receipt says 0', async () => { + const { p, engine } = await makeProtocol(); + await stage(p, [ + { type: 'seed', name: 'a_missing_object', item: missingObjectSeed }, + { type: 'seed', name: 'z_good', item: goodSeed(['R1', 'R2']) }, + ]); + + const res: any = await p.publishPackageDrafts({ packageId: PKG }); + + // The loader stops at the failing dataset: the good seed after it never ran. + expect(await rowsAtEngine(engine)).toEqual([]); + expect(res.seedApplied.success).toBe(false); + expect(res.seedApplied.inserted).toBe(0); + expect(res.seedApplied.updated).toBe(0); + expect(res.seedApplied.error).toContain(`'${MISSING}'`); + }); + + it('the only seed fails before anything is written: inserted 0, updated 0 — truthfully', async () => { + const { p, engine } = await makeProtocol(); + await stage(p, [{ type: 'seed', name: 'r_missing_object', item: missingObjectSeed }]); + + const res: any = await p.publishPackageDrafts({ packageId: PKG }); + + expect(await rowsAtEngine(engine)).toEqual([]); + expect(res.seedApplied.success).toBe(false); + expect(res.seedApplied.inserted).toBe(0); + expect(res.seedApplied.updated).toBe(0); + expect(res.seedApplied.error).toContain(`'${MISSING}'`); + }); +}); From 1c64cb68fc46c7a06c587c1611b08707d74e4e05 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 23:18:04 +0000 Subject: [PATCH 2/2] test: pin partial seed writes beside the existing seed-file doubles Moves the loader-level pins into seed-loader-seed-files.test.ts so they reuse its pinned engine double, and makes the publish-door test driver honour the caller's limit and type its find options. Claude-Session: https://claude.ai/code/session_01L6bhgUCizoYWQVJWjhdR6F Co-authored-by: Claude --- .../src/seed-loader-partial-writes.test.ts | 148 ------------------ .../src/seed-loader-seed-files.test.ts | 61 +++++++- ...package-drafts-seed-receipt-counts.test.ts | 7 +- 3 files changed, 64 insertions(+), 152 deletions(-) delete mode 100644 packages/metadata-protocol/src/seed-loader-partial-writes.test.ts diff --git a/packages/metadata-protocol/src/seed-loader-partial-writes.test.ts b/packages/metadata-protocol/src/seed-loader-partial-writes.test.ts deleted file mode 100644 index 66f7818591b..00000000000 --- a/packages/metadata-protocol/src/seed-loader-partial-writes.test.ts +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. - -/** - * `seedLoadPartialWrites` — what a load that THREW had already written. - * - * A load writes dataset by dataset and rolls nothing back when a later one - * throws, so the caller reporting the failure (the publish batch's - * `seedApplied` receipt) needs the rows that landed: the finished datasets' - * and those of the dataset that threw part-way. The publish-door pins live in - * `@objectstack/objectql`'s `publish-package-drafts-seed-receipt-counts.test.ts`; - * this file pins the one case that door cannot stage — a throw INSIDE a - * dataset after some of its rows were written (here: the file service failing - * while a written row's seed file is settled) — and the throw that comes - * before any dataset. - * - * Every count is checked against the fake engine's store. - */ - -import { createHash } from 'node:crypto'; -import { describe, it, expect, vi } from 'vitest'; -import type { IDataEngine, IFieldFileService, IMetadataService } from '@objectstack/spec/contracts'; -import { seedFile } from '@objectstack/spec/data'; -import { assertEngineDeleteDispatch, assertEngineUpdateDispatch, assertEngineFindOnePredicate } from '@objectstack/metadata-core'; -import { SeedLoaderService, seedLoadPartialWrites } from './seed-loader.js'; - -function createLogger() { - return { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }; -} - -function createEngine(): { engine: IDataEngine; store: Record } { - const store: Record = {}; - let next = 0; - const matches = (row: Record, where: Record) => - Object.entries(where).every(([k, v]) => { - if (k.startsWith('$')) throw new Error(`fake driver: unsupported operator ${k}`); - return row[k] === v; - }); - const engine = { - find: vi.fn(async (object: string, query?: any) => { - let rows = store[object] ?? []; - if (query?.where) rows = rows.filter((r) => matches(r, query.where)); - return rows.map((r) => ({ ...r })); - }), - findOne: vi.fn(async (object: string, query?: any) => { - assertEngineFindOnePredicate(object, query); - const rows = (store[object] ?? []).filter((r) => matches(r, query?.where ?? {})); - return rows[0] ? { ...rows[0] } : null; - }), - insert: vi.fn(async (object: string, data: any) => { - store[object] ??= []; - const one = (d: any) => { - const row = { id: `row-${++next}`, ...d }; - store[object].push(row); - return { ...row }; - }; - return Array.isArray(data) ? data.map(one) : one(data); - }), - update: vi.fn(async (object: string, data: any) => { - assertEngineUpdateDispatch(data, undefined); - const rows = store[object] ?? []; - const i = rows.findIndex((r) => r.id === data.id); - if (i >= 0) rows[i] = { ...rows[i], ...data }; - return data; - }), - delete: vi.fn(async (_object: string, options?: any) => { - assertEngineDeleteDispatch(options); - return { deleted: 1 }; - }), - count: vi.fn(async (object: string) => (store[object] ?? []).length), - aggregate: vi.fn(async () => []), - } as unknown as IDataEngine; - return { engine, store }; -} - -/** A `field-files` service that stores files and is DOWN for every read of them. */ -function createFieldFilesDownOnRead(): IFieldFileService { - let next = 0; - return { - storeFile: vi.fn(async (input: any) => ({ fileId: `file-${++next}`, size: input.data.byteLength })), - describeFiles: vi.fn(async () => { throw new Error('the storage service is down'); }), - claimFiles: vi.fn(async () => {}), - } as unknown as IFieldFileService; -} - -/** `pw_tag` (plain) is loaded first: `pw_doc` looks it up. */ -function createMetadata(fail?: Error): IMetadataService { - const objects: Record = { - pw_tag: { name: 'pw_tag', fields: { name: { type: 'text' } } }, - pw_doc: { - name: 'pw_doc', - fields: { code: { type: 'text' }, file: { type: 'file' }, tag: { type: 'lookup', reference: 'pw_tag' } }, - }, - }; - return { - getObject: vi.fn(async (name: string) => { if (fail) throw fail; return objects[name]; }), - } as unknown as IMetadataService; -} - -const CONFIG = { dryRun: false, haltOnError: false, multiPass: true, defaultMode: 'upsert', batchSize: 1000, transaction: false } as any; - -const pdf = Buffer.from('%PDF pw'); -const FILES = { - 'fixtures/a.pdf': { - contentType: 'application/pdf', - size: pdf.byteLength, - sha256: createHash('sha256').update(pdf).digest('hex'), - data: pdf.toString('base64'), - }, -}; - -const seeds = () => [ - { object: 'pw_doc', externalId: 'code', mode: 'upsert', env: ['prod', 'dev', 'test'], files: FILES, - records: [{ code: 'D1', tag: 'T1', file: seedFile('fixtures/a.pdf') }] }, - { object: 'pw_tag', mode: 'upsert', env: ['prod', 'dev', 'test'], records: [{ name: 'T1' }, { name: 'T2' }] }, -] as any; - -describe('seedLoadPartialWrites', () => { - it('counts the finished datasets AND the rows the throwing dataset wrote, naming its object', async () => { - const { engine, store } = createEngine(); - const loader = new SeedLoaderService(engine, createMetadata(), createLogger() as any, { - getFieldFiles: () => createFieldFilesDownOnRead(), - }); - - const err = await loader.load({ seeds: seeds(), config: CONFIG }).then(() => undefined, (e: unknown) => e); - - expect((err as Error).message).toBe('the storage service is down'); - // At the engine: both tags, and the doc row written before its file settle threw. - expect((store.pw_tag ?? []).map((r) => r.name).sort()).toEqual(['T1', 'T2']); - expect((store.pw_doc ?? []).map((r) => r.code)).toEqual(['D1']); - expect(seedLoadPartialWrites(err)).toEqual({ inserted: 3, updated: 0, failedObject: 'pw_doc' }); - }); - - it('a load that throws before any dataset wrote: zero, and no object named', async () => { - const { engine, store } = createEngine(); - const loader = new SeedLoaderService(engine, createMetadata(new Error('metadata store unavailable')), createLogger() as any); - - const err = await loader.load({ seeds: seeds(), config: CONFIG }).then(() => undefined, (e: unknown) => e); - - expect((err as Error).message).toBe('metadata store unavailable'); - expect(store).toEqual({}); - expect(seedLoadPartialWrites(err)).toEqual({ inserted: 0, updated: 0 }); - }); - - it('an error that did not come out of a load carries no reading', () => { - expect(seedLoadPartialWrites(new Error('elsewhere'))).toBeUndefined(); - expect(seedLoadPartialWrites('a string')).toBeUndefined(); - }); -}); diff --git a/packages/metadata-protocol/src/seed-loader-seed-files.test.ts b/packages/metadata-protocol/src/seed-loader-seed-files.test.ts index 968448d2ac5..56d8b0263a8 100644 --- a/packages/metadata-protocol/src/seed-loader-seed-files.test.ts +++ b/packages/metadata-protocol/src/seed-loader-seed-files.test.ts @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'; import { describe, it, expect, vi } from 'vitest'; -import { SeedLoaderService } from './seed-loader.js'; +import { SeedLoaderService, seedLoadPartialWrites } from './seed-loader.js'; import type { FieldFileDescription, FieldFileSlot, @@ -518,3 +518,62 @@ describe('a seed of many files yields to waiting requests (#26670)', () => { expect(Math.max(...sizes)).toBeLessThanOrEqual(20); }); }); + +/** + * `seedLoadPartialWrites` — what a load that THREW had already written. + * + * A load writes dataset by dataset and rolls nothing back when a later one + * throws, so the publish batch's `seedApplied` receipt reads the rows that + * landed off the error: the finished datasets' and those of the dataset that + * threw part-way. The publish-door pins live in `@objectstack/objectql`'s + * `publish-package-drafts-seed-receipt-counts.test.ts`; these pin what that + * door cannot stage — a throw INSIDE a dataset after one of its rows was + * written (the file service failing while that row's seed file is settled) — + * and a throw before any dataset ran. Counts are checked against the store. + */ +describe('seedLoadPartialWrites: the rows a thrown load had written', () => { + /** `pw_tag` loads first (`pw_doc` looks it up); neither looks itself up. */ + function partialMetadata(fail?: Error): IMetadataService { + const objects: Record = { + pw_tag: { name: 'pw_tag', fields: { name: { type: 'text' } } }, + pw_doc: { name: 'pw_doc', fields: { code: { type: 'text' }, file: { type: 'file' }, tag: { type: 'lookup', reference: 'pw_tag' } } }, + }; + return { + getObject: vi.fn(async (name: string) => { if (fail) throw fail; return objects[name]; }), + } as unknown as IMetadataService; + } + const partialSeeds = () => [ + dataset([{ code: 'D1', tag: 'T1', file: seedFile('fixtures/v1.pdf') }], undefined, { object: 'pw_doc' }), + { object: 'pw_tag', mode: 'upsert', env: ['prod', 'dev', 'test'], records: [{ name: 'T1' }, { name: 'T2' }] }, + ] as any; + + it('counts the finished datasets AND the rows the throwing dataset wrote, naming its object', async () => { + const { engine, store } = createEngine(); + const { service } = createFieldFiles(); + service.describeFiles = vi.fn(async () => { throw new Error('the storage service is down'); }); + const seedLoader = new SeedLoaderService(engine, partialMetadata(), createLogger(), { getFieldFiles: () => service }); + + const err = await seedLoader.load({ seeds: partialSeeds(), config: CONFIG }).then(() => undefined, (e: unknown) => e); + + expect((err as Error).message).toBe('the storage service is down'); + expect((store.pw_tag ?? []).map((r) => r.name).sort()).toEqual(['T1', 'T2']); + expect((store.pw_doc ?? []).map((r) => r.code)).toEqual(['D1']); + expect(seedLoadPartialWrites(err)).toEqual({ inserted: 3, updated: 0, failedObject: 'pw_doc' }); + }); + + it('a load that throws before any dataset wrote: zero, and no object named', async () => { + const { engine, store } = createEngine(); + const seedLoader = new SeedLoaderService(engine, partialMetadata(new Error('metadata store unavailable')), createLogger()); + + const err = await seedLoader.load({ seeds: partialSeeds(), config: CONFIG }).then(() => undefined, (e: unknown) => e); + + expect((err as Error).message).toBe('metadata store unavailable'); + expect(store).toEqual({}); + expect(seedLoadPartialWrites(err)).toEqual({ inserted: 0, updated: 0 }); + }); + + it('an error that did not come out of a load carries no reading', () => { + expect(seedLoadPartialWrites(new Error('elsewhere'))).toBeUndefined(); + expect(seedLoadPartialWrites('a string')).toBeUndefined(); + }); +}); diff --git a/packages/objectql/src/publish-package-drafts-seed-receipt-counts.test.ts b/packages/objectql/src/publish-package-drafts-seed-receipt-counts.test.ts index 60e62236f32..43ed0ac43b1 100644 --- a/packages/objectql/src/publish-package-drafts-seed-receipt-counts.test.ts +++ b/packages/objectql/src/publish-package-drafts-seed-receipt-counts.test.ts @@ -82,7 +82,8 @@ function makeMemoryDriver() { async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, async find(object: string, ast: any) { - return Array.from(storeFor(object).values()).filter((r) => matchesWhere(r, ast?.where)); + const rows = Array.from(storeFor(object).values()).filter((r) => matchesWhere(r, ast?.where)); + return typeof ast?.limit === 'number' ? rows.slice(0, ast.limit) : rows; }, async findOne(object: string, ast: any) { for (const r of storeFor(object).values()) if (matchesWhere(r, ast?.where)) return r; @@ -159,7 +160,7 @@ async function stage( /** The seeded object's rows, read at the engine. */ async function rowsAtEngine(engine: ObjectQL): Promise { - const rows = await engine.find(ITEM, { context: { isSystem: true } } as any); + const rows = await engine.find(ITEM, { context: { isSystem: true } }); return (rows as Array<{ name?: string }>).map((r) => String(r.name)).sort(); } @@ -207,7 +208,7 @@ describe('publish-drafts seedApplied counts what was written', () => { const res: any = await p.publishPackageDrafts({ packageId: PKG }); - const rows = await engine.find(ITEM, { context: { isSystem: true } } as any) as Array<{ name: string; label?: string }>; + const rows = await engine.find(ITEM, { context: { isSystem: true } }) as Array<{ name: string; label?: string }>; expect(rows.map((r) => `${r.name}:${r.label}`).sort()).toEqual(['R1:one', 'R2:two']); expect(res.seedApplied.success).toBe(false); expect(res.seedApplied.updated).toBe(2);