fix(metadata-protocol): a failed seed in a package publish counts the rows earlier seeds wrote

.changeset/27393-seed-receipt-partial-counts.md

@@ -0,0 +1,18 @@
+---
+'@objectstack/metadata-protocol': patch
+---
+
+fix(metadata-protocol): when a seed in a package publish fails, the `seedApplied` receipt counts the rows the earlier seeds wrote and names the seed that stopped the load
+
+Clause-②: no
+Metadata-protocol: none
+
+`POST /api/v1/packages/:id/publish-drafts` applies every published seed body in one load. The load writes seed by seed and stops at the first one that throws (for example, a seed naming an object that does not exist); the rows the seeds ahead of it wrote stay written. The receipt used to answer `inserted: 0, updated: 0` all the same, so an author or Studio reading it was told nothing had been written while the table held those rows.
+
+Now the receipt keeps its shape and its counters describe what was written:
+
+- `inserted` / `updated` are the rows the load wrote before it stopped, including any rows the failing seed itself wrote before it threw.
+- `success` is `false`, as before.
+- `error` names the seed that stopped the load by the object its dataset populates: `seed apply stopped at the seed for object 'OBJECT': ...`, followed by the same sentence as before (a refusal the error declared is quoted, anything else is withheld as `seed apply failed`).
+
+Unchanged: which seeds are published, that the seeds after the failing one are not applied, and that nothing is rolled back. A throw before the loader writes anything (a malformed seed body, for example) still answers `inserted: 0, updated: 0` and the same `error` as before.

packages/metadata-protocol/src/protocol-publish-batch.ts

@@ -11,6 +11,8 @@
  * `ObjectStackProtocolImplementation` method of the same name as it stood,
  * re-indented one level and otherwise unedited: the same reads, in the same
  * order, answering with the same receipts and the same degradations.
+ * Edited since the move: `applySeedBodies`' failure receipt counts the rows the
+ * loader wrote before it threw and names the seed that stopped it.
  *
  * ## How the moved bodies reach the class
  *
@@ -46,6 +48,7 @@ import {
     seedRequestValidationError,
 } from './protocol.js';
 import { CLOSURE_CONTEXT_KEY_BY_TYPE, type RuntimePendingDeclarations } from './runtime-authoring-gate.js';
+import type { SeedLoadPartialWrites } from './seed-loader.js';
 
 /**
  * The `ObjectStackProtocolImplementation` members the functions below read —
@@ -72,14 +75,18 @@ export async function applySeedBodies(
     success: boolean; inserted: number; updated: number; error?: string; errors?: unknown[];
     issues?: Array<{ path: string; message: string; code?: string | undefined }>;
 }> {
+    // The loader's reader of what a THROWN load had written. Unset until the
+    // loader module is imported, which no write precedes.
+    let partialWritesOf: ((e: unknown) => SeedLoadPartialWrites | undefined) | undefined;
     try {
         const seeds = bodies.filter(
             (b: any) => b && typeof b.object === 'string' && Array.isArray(b.records),
         );
         if (seeds.length === 0) {
             return { success: false, inserted: 0, updated: 0, error: 'seed apply: no readable seed bodies' };
         }
-        const { SeedLoaderService } = await import('./seed-loader.js');
+        const { SeedLoaderService, seedLoadPartialWrites } = await import('./seed-loader.js');
+        partialWritesOf = seedLoadPartialWrites;
         const { SeedLoaderRequestSchema } = await import('@objectstack/spec/data');
         // The loader only needs `getObject` from IMetadataService (dependency
         // graph + field introspection); satisfy it from the protocol's own
@@ -134,9 +141,21 @@ export async function applySeedBodies(
         // which key. That declaration is the whole reason this site could
         // be converted rather than escalated.
         console.warn(`[protocol.applySeedBodies] seed apply failed: ${e?.message ?? e}`);
+        // The counters describe what was WRITTEN. The loader writes dataset
+        // by dataset and stops at the one that throws; the rows of the
+        // datasets ahead of it stay written (nothing rolls them back), so the
+        // receipt carries the loader's own count of them and names the seed
+        // that stopped it. No reading means the throw came before the loader
+        // wrote anything — the request parse, or the loader's import.
+        const written = partialWritesOf?.(e);
+        const failure = clientFacingFailureText(e, 'seed apply failed');
         return {
-            success: false, inserted: 0, updated: 0,
-            error: clientFacingFailureText(e, 'seed apply failed'),
+            success: false,
+            inserted: written?.inserted ?? 0,
+            updated: written?.updated ?? 0,
+            error: written?.failedObject
+                ? `seed apply stopped at the seed for object '${written.failedObject}': ${failure}`
+                : failure,
             // [#10524] A DECLARED refusal's structured findings ride the
             // receipt beside the headline `error` — the message is a
             // one-sentence headline now, so this is where the per-key

packages/metadata-protocol/src/seed-loader-seed-files.test.ts

@@ -2,7 +2,7 @@
 
 import { createHash } from 'node:crypto';
 import { describe, it, expect, vi } from 'vitest';
-import { SeedLoaderService } from './seed-loader.js';
+import { SeedLoaderService, seedLoadPartialWrites } from './seed-loader.js';
 import type {
   FieldFileDescription,
   FieldFileSlot,
@@ -518,3 +518,62 @@ describe('a seed of many files yields to waiting requests (#26670)', () => {
     expect(Math.max(...sizes)).toBeLessThanOrEqual(20);
   });
 });
+
+/**
+ * `seedLoadPartialWrites` — what a load that THREW had already written.
+ *
+ * A load writes dataset by dataset and rolls nothing back when a later one
+ * throws, so the publish batch's `seedApplied` receipt reads the rows that
+ * landed off the error: the finished datasets' and those of the dataset that
+ * threw part-way. The publish-door pins live in `@objectstack/objectql`'s
+ * `publish-package-drafts-seed-receipt-counts.test.ts`; these pin what that
+ * door cannot stage — a throw INSIDE a dataset after one of its rows was
+ * written (the file service failing while that row's seed file is settled) —
+ * and a throw before any dataset ran. Counts are checked against the store.
+ */
+describe('seedLoadPartialWrites: the rows a thrown load had written', () => {
+  /** `pw_tag` loads first (`pw_doc` looks it up); neither looks itself up. */
+  function partialMetadata(fail?: Error): IMetadataService {
+    const objects: Record<string, any> = {
+      pw_tag: { name: 'pw_tag', fields: { name: { type: 'text' } } },
+      pw_doc: { name: 'pw_doc', fields: { code: { type: 'text' }, file: { type: 'file' }, tag: { type: 'lookup', reference: 'pw_tag' } } },
+    };
+    return {
+      getObject: vi.fn(async (name: string) => { if (fail) throw fail; return objects[name]; }),
+    } as unknown as IMetadataService;
+  }
+  const partialSeeds = () => [
+    dataset([{ code: 'D1', tag: 'T1', file: seedFile('fixtures/v1.pdf') }], undefined, { object: 'pw_doc' }),
+    { object: 'pw_tag', mode: 'upsert', env: ['prod', 'dev', 'test'], records: [{ name: 'T1' }, { name: 'T2' }] },
+  ] as any;
+
+  it('counts the finished datasets AND the rows the throwing dataset wrote, naming its object', async () => {
+    const { engine, store } = createEngine();
+    const { service } = createFieldFiles();
+    service.describeFiles = vi.fn(async () => { throw new Error('the storage service is down'); });
+    const seedLoader = new SeedLoaderService(engine, partialMetadata(), createLogger(), { getFieldFiles: () => service });
+
+    const err = await seedLoader.load({ seeds: partialSeeds(), config: CONFIG }).then(() => undefined, (e: unknown) => e);
+
+    expect((err as Error).message).toBe('the storage service is down');
+    expect((store.pw_tag ?? []).map((r) => r.name).sort()).toEqual(['T1', 'T2']);
+    expect((store.pw_doc ?? []).map((r) => r.code)).toEqual(['D1']);
+    expect(seedLoadPartialWrites(err)).toEqual({ inserted: 3, updated: 0, failedObject: 'pw_doc' });
+  });
+
+  it('a load that throws before any dataset wrote: zero, and no object named', async () => {
+    const { engine, store } = createEngine();
+    const seedLoader = new SeedLoaderService(engine, partialMetadata(new Error('metadata store unavailable')), createLogger());
+
+    const err = await seedLoader.load({ seeds: partialSeeds(), config: CONFIG }).then(() => undefined, (e: unknown) => e);
+
+    expect((err as Error).message).toBe('metadata store unavailable');
+    expect(store).toEqual({});
+    expect(seedLoadPartialWrites(err)).toEqual({ inserted: 0, updated: 0 });
+  });
+
+  it('an error that did not come out of a load carries no reading', () => {
+    expect(seedLoadPartialWrites(new Error('elsewhere'))).toBeUndefined();
+    expect(seedLoadPartialWrites('a string')).toBeUndefined();
+  });
+});

packages/metadata-protocol/src/seed-loader.ts

@@ -558,6 +558,35 @@ interface ReplayComparands {
   readonly masked: ReadonlySet<string>;
 }
 
+/**
+ * What a {@link SeedLoaderService.load} that THREW had already written.
+ *
+ * A load writes dataset by dataset and nothing undoes a dataset's rows when a
+ * later one throws, so a caller that reports the failure must still be able to
+ * say what landed: a receipt answering `inserted: 0` while the table holds the
+ * earlier datasets' rows is a receipt that lies. Read with
+ * {@link seedLoadPartialWrites} on the error the load threw.
+ */
+export interface SeedLoadPartialWrites {
+  /** Rows inserted before the load threw — finished datasets plus the one that threw. */
+  readonly inserted: number;
+  /** Rows updated before the load threw, counted the same way. */
+  readonly updated: number;
+  /** The object of the dataset that was loading when the load threw; absent when it threw outside any dataset. */
+  readonly failedObject?: string;
+}
+
+/** The writes a thrown load had made, keyed by the error it threw. */
+const partialWritesByError = new WeakMap<object, SeedLoadPartialWrites>();
+
+/**
+ * What the {@link SeedLoaderService.load} that threw `err` had written, or
+ * `undefined` when `err` did not come out of a load.
+ */
+export function seedLoadPartialWrites(err: unknown): SeedLoadPartialWrites | undefined {
+  return err !== null && typeof err === 'object' ? partialWritesByError.get(err) : undefined;
+}
+
 /** What a {@link SeedLoaderService} is built with beyond its engine, metadata and logger. */
 export interface SeedLoaderServiceOptions {
   /**
@@ -703,6 +732,17 @@ export class SeedLoaderService implements ISeedLoaderService {
    */
   private uncomparedCredentials = new Map<string, { fields: Set<string>; rows: number }>();
 
+  /**
+   * The CURRENT {@link load}'s writes so far, for {@link SeedLoadPartialWrites}:
+   * the finished datasets' results, and the counters of the dataset loading
+   * now. Reset per `load`; an instance field for {@link summariesStale}'s
+   * reason (datasets load strictly sequentially).
+   */
+  private loadProgress?: {
+    finished: readonly SeedLoadResultParsed[];
+    loading?: { object: string; written: { inserted: number; updated: number } };
+  };
+
   /**
    * [#25779] The `field-files` service (`FIELD_FILE_SERVICE`) a seed file is
    * stored through, read per dataset. Absent — a loader built without one, or
@@ -744,10 +784,34 @@ export class SeedLoaderService implements ISeedLoaderService {
    * it.
    */
   async load(request: SeedLoaderRequestParsed): Promise<SeedLoaderResultParsed> {
-    return runWithAdvisoryAggregation(
-      () => this.loadDatasets(request),
-      (groups) => this.reportAdvisorySummary(groups),
-    );
+    this.loadProgress = undefined;
+    try {
+      return await runWithAdvisoryAggregation(
+        () => this.loadDatasets(request),
+        (groups) => this.reportAdvisorySummary(groups),
+      );
+    } catch (err) {
+      throw this.notePartialWrites(err);
+    }
+  }
+
+  /**
+   * Record on `err` what this load wrote before it threw (see
+   * {@link SeedLoadPartialWrites}) and hand back the error to rethrow — the
+   * same error, so its declared status and message reach the caller as they
+   * did; a thrown non-object is wrapped, since only an object can carry it.
+   */
+  private notePartialWrites(err: unknown): unknown {
+    const finished = this.loadProgress?.finished ?? [];
+    const loading = this.loadProgress?.loading;
+    const partial: SeedLoadPartialWrites = {
+      inserted: finished.reduce((sum, r) => sum + r.inserted, 0) + (loading?.written.inserted ?? 0),
+      updated: finished.reduce((sum, r) => sum + r.updated, 0) + (loading?.written.updated ?? 0),
+      ...(loading ? { failedObject: loading.object } : {}),
+    };
+    const carrier = err !== null && typeof err === 'object' ? err : new Error(String(err));
+    partialWritesByError.set(carrier, partial);
+    return carrier;
   }
 
   /**
@@ -852,6 +916,7 @@ export class SeedLoaderService implements ISeedLoaderService {
     this.warnOnUnresolvedLocaleScope(config, request.seeds);
     const allErrors: ReferenceResolutionError[] = [];
     const allResults: SeedLoadResultParsed[] = [];
+    this.loadProgress = { finished: allResults };
     // Per-load counter — a service instance can be reused across loads.
     this.summariesStale = 0;
     // [#9071] Same per-load lifetime, same reason: a publish between two loads
@@ -1127,8 +1192,9 @@ export class SeedLoaderService implements ISeedLoaderService {
     // `matchesExistingRow` in the record loop below.
     const createOnlyColumns = createOnlySeedColumns(dataset, externalId);
 
-    let inserted = 0;
-    let updated = 0;
+    // Read by `notePartialWrites` should this dataset throw part-way.
+    const written = { inserted: 0, updated: 0 };
+    if (this.loadProgress) this.loadProgress.loading = { object: objectName, written };
     let skipped = 0;
     let errored = 0;
     let referencesResolved = 0;
@@ -1342,7 +1408,7 @@ export class SeedLoaderService implements ISeedLoaderService {
       for (const res of writeResults) {
         const { recordIndex, externalIdValue, record } = batch[res.index];
         if (res.ok) {
-          inserted++;
+          written.inserted++;
           const internalId = this.extractId(res.record);
           if (internalId) noteLanded(recordIndex, internalId); // [commit 9a884c6e4]
           if (externalIdValue && internalId) {
@@ -2046,8 +2112,8 @@ export class SeedLoaderService implements ISeedLoaderService {
               objectName, record, mode, externalId, existingRecords, comparands
             );
 
-            if (result.action === 'inserted') inserted++;
-            else if (result.action === 'updated') updated++;
+            if (result.action === 'inserted') written.inserted++;
+            else if (result.action === 'updated') written.updated++;
             else if (result.action === 'skipped') {
               skipped++;
               if (result.uncompared) this.noteUncomparedCredentials(objectName, result.uncompared); // [#24624]
@@ -2110,7 +2176,7 @@ export class SeedLoaderService implements ISeedLoaderService {
             }
             try {
               await this.writeRecoveringSummary(objectName, () => withTransientRetry(() => this.engine.update(objectName, { ...record, id: decision.id }, opts)));
-              updated++;
+              written.updated++;
               await settleSeedFiles(i, decision.id); // [#25779]
             } catch (err: any) {
               errored++;
@@ -2142,7 +2208,7 @@ export class SeedLoaderService implements ISeedLoaderService {
         if (externalIdValue) {
           insertedRecords.get(objectName)!.set(externalIdValue, `dry-run-id-${i}`);
         }
-        inserted++; // Count as "would be inserted"
+        written.inserted++; // Count as "would be inserted"
       }
     }
 
@@ -2176,11 +2242,12 @@ export class SeedLoaderService implements ISeedLoaderService {
         .map(([, id]) => id),
     )];
 
+    if (this.loadProgress) this.loadProgress.loading = undefined;
     return {
       object: objectName,
       mode,
-      inserted,
-      updated,
+      inserted: written.inserted,
+      updated: written.updated,
       skipped,
       errored,
       total: dataset.records.length,