From 8ec42d45505c2ff4dc1d9a62905a9b887c2c6f4a Mon Sep 17 00:00:00 2001 From: Bernardo Donadio Date: Sun, 25 Oct 2026 00:21:05 -0300 Subject: [PATCH 1/2] fix: classify cutover-set activation like single Set activation repeated three defects the single-generation activation already fixed (#1617, #1618, #1648). Scratch: activateCutoverSet created its scratch directory before any classification, so every early refusal touched the disk and an uninitialized home threw ENOENT. The directory is now allocated per member census through deps.allocateScratch, after the S7 identity check, and removed in a finally. The set prefix is kept by overriding the default allocator. An allocation failure marks the member unverifiable (state-unresolvable) so the S8 gate names it. An errno from the machine identity read is refused as state-unresolvable at S0. Target failures: identityCheck rethrew a StorageOperationError from the S0, S5 and S7 identity queries, and targetFailure missed the PortableStreamError(source-unavailable) wrapper of the census reader. Both now classify as target-unhealthy or target-unresolvable on the member and the gate refuses (or interrupts after a checkpoint). Every call site records the failure and continues, so a later member's checkpoint conflict still surfaces. Cancellation and other errors still reject. Applied: classifyApplied derived yes and no from the journal phase alone. It now mirrors the single-generation rules, comparing the live config and project map digests with the published or expected witnesses, or with the artifact's source witness when no journal of this set exists; in-flight phases and any readback failure are unknown. digestsMatch, recoveryFileDigests and APPLIED_IN_FLIGHT_PHASES are exported from activate-generation.ts (visibility only). Existing assertions that expected a phase-only no are updated to unknown where no source witness exists. Signed-off-by: Bernardo Donadio --- ...3-cutover-set-activation-classification.md | 9 + docs/migration-cutover-set.md | 11 + src/migration/activate-generation.ts | 6 +- src/migration/cutover-set-activate.ts | 144 +++++--- ...-cutover-set-activation-classification.yml | 11 + ...utover-set-activate-classification.test.ts | 318 ++++++++++++++++++ .../cutover-set-activate-faults.test.ts | 6 +- 7 files changed, 453 insertions(+), 52 deletions(-) create mode 100644 .changeset/1673-cutover-set-activation-classification.md create mode 100644 test/codecov-ownership/1673-cutover-set-activation-classification.yml create mode 100644 test/migration/cutover-set-activate-classification.test.ts diff --git a/.changeset/1673-cutover-set-activation-classification.md b/.changeset/1673-cutover-set-activation-classification.md new file mode 100644 index 000000000..e9066ae40 --- /dev/null +++ b/.changeset/1673-cutover-set-activation-classification.md @@ -0,0 +1,9 @@ +--- +"@donadiosolutions/lcm": patch +--- + +Cutover-set activation now allocates census scratch only for a member census, +reports a storage failure of a destination identity query or census read as a +target refusal naming the member, and reports an interrupted run as applied +only when the live configuration and project map agree with the publication +journal or the set's pre-publication source. diff --git a/docs/migration-cutover-set.md b/docs/migration-cutover-set.md index a1231dcbc..2659a5219 100644 --- a/docs/migration-cutover-set.md +++ b/docs/migration-cutover-set.md @@ -290,6 +290,17 @@ call) and the coverage check may have reconciled the project map. No member's evidence is ever inferred from another's, and no time, count or heuristic decides eligibility. +The set activation follows the single-generation rules for scratch space, +target failures and the `applied` answer of an interruption (see +[Migration activation](migration-cutover.md#migration-activation)). A member's +census scratch directory is allocated only when that member's census is read +and is removed afterwards, so an early refusal leaves nothing on disk. A +storage failure of a member's destination identity query or census read is +attributed to that member as `target-unhealthy` or `target-unresolvable`, and +the set is refused naming it. An interrupted outcome reports `applied` as `yes` +or `no` only when the live configuration and project map agree with the +publication journal or the set's pre-publication source, otherwise `unknown`. + ### One checkpoint, member effects, one selection, one publication Once every member passes, the publication material is composed once for all diff --git a/src/migration/activate-generation.ts b/src/migration/activate-generation.ts index 0bc893e95..eddc7ee7d 100644 --- a/src/migration/activate-generation.ts +++ b/src/migration/activate-generation.ts @@ -800,14 +800,14 @@ function projectBinding(ctx: Context): ProjectBinding { /** The own journal records the config as published: yes needs the live selection to match it. */ const APPLIED_YES_PHASES: readonly BackendPublicationJournal["phase"][] = ["config-published", "releasing", "released", "completed"]; /** The write may be in flight: never proven either way, even when the live selection equals the source. */ -const APPLIED_IN_FLIGHT_PHASES: readonly BackendPublicationJournal["phase"][] = ["map-publishing", "map-published", "config-publishing"]; +export const APPLIED_IN_FLIGHT_PHASES: readonly BackendPublicationJournal["phase"][] = ["map-publishing", "map-published", "config-publishing"]; -function digestsMatch(live: MigrationActivationSelectionDigests, configSha256: string, projectMapSha256: string): boolean { +export function digestsMatch(live: MigrationActivationSelectionDigests, configSha256: string, projectMapSha256: string): boolean { return live.configSha256 === configSha256 && live.projectMapSha256 === projectMapSha256; } /** The digest of a recovery file under the journal convention: raw bytes for the config, canonical JSON for the map, {} when absent. */ -function recoveryFileDigests(config: BackendPublicationRecoveryFile, projectMap: BackendPublicationRecoveryFile): MigrationActivationSelectionDigests { +export function recoveryFileDigests(config: BackendPublicationRecoveryFile, projectMap: BackendPublicationRecoveryFile): MigrationActivationSelectionDigests { return { configSha256: createHash("sha256").update(config.presence === "present" ? config.content : "{}").digest("hex"), projectMapSha256: backendPublicationCanonicalSha256( diff --git a/src/migration/cutover-set-activate.ts b/src/migration/cutover-set-activate.ts index d6ab6bf7d..13edc0c6a 100644 --- a/src/migration/cutover-set-activate.ts +++ b/src/migration/cutover-set-activate.ts @@ -21,6 +21,7 @@ import { import type { ComposeBackendPublicationMaterialResult } from "../storage/backend-publication-material.js"; import type { StorageIdentityContext } from "../storage/contracts.js"; import { StorageOperationError } from "../storage/errors.js"; +import { PortableStreamError } from "../storage/portable-record.js"; import type { PostgreSqlConnectionSettings, PostgreSqlQueryExecutor } from "../storage/postgresql/contracts.js"; import { connectionSettings } from "../storage/postgresql/factory.js"; import { @@ -28,7 +29,9 @@ import { POSTGRESQL_BACKEND_PUBLICATION_RESOURCE_TYPE, } from "../storage/postgresql/publication-guard.js"; import { + APPLIED_IN_FLIGHT_PHASES, createMigrationActivationDependencies, + digestsMatch, MigrationActivationError, type MigrationActivationDependencies, type MigrationActivationInterruptionCause, @@ -37,6 +40,7 @@ import { type MigrationActivationRefusalReason, type MigrationActivationRefusalStep, type MigrationActivationSnapshot, + recoveryFileDigests, type MigrationActivationTriState, } from "./activate-generation.js"; import { @@ -284,6 +288,7 @@ export type CutoverSetActivateDependencies = MigrationActivationDependencies & R export function createCutoverSetActivateDependencies(input: Readonly<{ homeDir: string }>): CutoverSetActivateDependencies { return { ...createMigrationActivationDependencies(input), + allocateScratch: (home) => mkdtempSync(join(lcmHomeDir(home), "cutover-set-activation-scratch-")), resolveAuthorities: resolveCutoverSetMemberAuthorities, readCanonicalPath: readProjectMapCanonicalPath, resolveExpectedIdentity: resolveMigrationExpectedIdentity, @@ -354,7 +359,6 @@ type Context = { readonly coordinates: NonSecretPostgreSqlTarget; readonly maxProbeOrdinal: number; readonly now: () => Date; - readonly scratchParent: string; machineId: string; token: BackendPublicationLockToken; manifest: BackendCutoverSetManifest | null; @@ -422,20 +426,41 @@ function refused( return { exit: { status: "refused", setId: ctx.setId, step, reason, detail, failures, members: memberResults(ctx), effectBegun: false } }; } +/** The own journal records the config as published: yes needs the live selection to match it. */ const APPLIED_YES_PHASES: readonly BackendPublicationJournal["phase"][] = ["config-published", "releasing", "released", "completed"]; +/** "no" only when the live selection still equals the set's pre-publication source witness; otherwise "unknown". */ +function classifyAgainstSource(ctx: Context): MigrationActivationTriState { + // The witness enters the context at compose or at the stored artifact load; + // an interruption before either has no source proof. + const source = ctx.material?.material.source ?? ctx.artifact?.source; + if (source === undefined) return "unknown"; + const witness = recoveryFileDigests(source.config, source.projectMap); + return digestsMatch(ctx.deps.readSelectionDigests(ctx.homeDir), witness.configSha256, witness.projectMapSha256) ? "no" : "unknown"; +} + +/** + * applied is evidence-derived: the journal phase says what was attempted, and + * an independent readback of the live config and project map must agree + * before it is reported (the single-generation rules of activate-generation.ts). + * Everything else, including any readback failure, is unknown. Never throws. + */ function classifyApplied(ctx: Context): MigrationActivationTriState { - let journal: BackendPublicationJournal | null; try { const maintenance = ctx.deps.readMaintenanceJournal(ctx.homeDir); - if (maintenance !== null) return maintenance.sourceBackend === "sqlite" ? "no" : "unknown"; - journal = ctx.deps.readPublicationJournal(ctx.homeDir); + if (maintenance !== null) return maintenance.sourceBackend === "sqlite" ? classifyAgainstSource(ctx) : "unknown"; + const journal = ctx.deps.readPublicationJournal(ctx.homeDir); + if (journal === null || journal.publicationId !== ctx.setId) return classifyAgainstSource(ctx); + if (APPLIED_IN_FLIGHT_PHASES.includes(journal.phase)) return "unknown"; + const live = ctx.deps.readSelectionDigests(ctx.homeDir); + if (APPLIED_YES_PHASES.includes(journal.phase)) { + const { publishedConfigSha256, publishedProjectMapSha256 } = journal; + return publishedConfigSha256 !== null && publishedProjectMapSha256 !== null && digestsMatch(live, publishedConfigSha256, publishedProjectMapSha256) ? "yes" : "unknown"; + } + return digestsMatch(live, journal.expectedConfigSha256, journal.expectedProjectMapSha256) ? "no" : "unknown"; } catch { return "unknown"; } - if (journal === null || journal.publicationId !== ctx.setId) return "no"; - if (APPLIED_YES_PHASES.includes(journal.phase)) return "yes"; - return "unknown"; } function interrupted( @@ -492,37 +517,32 @@ export async function activateCutoverSet( // The coordinates helper refuses only through its typed MigrationIdentityError. throw new MigrationActivationError("invalid-input", "cutover-set activation target is invalid: " + (error as Error).message); } - const scratchParent = mkdtempSync(join(lcmHomeDir(input.homeDir), "cutover-set-activation-scratch-")); const ctx: Context = { input, deps, homeDir: input.homeDir, setId: input.confirmSetId, settings: connectionSettings(input.targetConfig), expectedOwner: input.targetConfig.postgresql.migrationRole, coordinates, maxProbeOrdinal: input.maxProbeOrdinal ?? DEFAULT_MAX_PROBE_ORDINAL, - now: input.now ?? (() => new Date()), scratchParent, + now: input.now ?? (() => new Date()), machineId: "", token: {} as BackendPublicationLockToken, manifest: null, members: [], journal: null, records: null, checkpoint: null, selection: null, artifact: null, material: null, identityDigest: null, ordinary: null, guard: null, coordinator: null, effectBegun: false, resumedFrom: "fresh", }; - try { - // The append barrier itself refuses a maintenance-entering journal, so that - // row of the routing table is classified before the barrier (as the - // preparation classifies its entry); every other row is read again under it. - const entering = classifyEnteringBeforeBarrier(ctx); - if (entering !== null) return entering; - return await deps.withAppendBarrier(input.homeDir, async (token) => { - ctx.token = token; - try { - return await withPreEffectLeasesReleased(ctx, await runUnderBarrier(ctx)); - } catch (error) { - // Best effort, only when readback proves no member effect began: the original error is the outcome. - if (ctx.guard !== null && headsPositivelyUnbegun(ctx)) await releaseAcquiredLeases(ctx); - throw error; - } finally { - if (ctx.ordinary !== null) await ctx.ordinary.close(); - } - }); - } finally { - rmSync(scratchParent, { recursive: true, force: true }); - } + // The append barrier itself refuses a maintenance-entering journal, so that + // row of the routing table is classified before the barrier (as the + // preparation classifies its entry); every other row is read again under it. + const entering = classifyEnteringBeforeBarrier(ctx); + if (entering !== null) return entering; + return await deps.withAppendBarrier(input.homeDir, async (token) => { + ctx.token = token; + try { + return await withPreEffectLeasesReleased(ctx, await runUnderBarrier(ctx)); + } catch (error) { + // Best effort, only when readback proves no member effect began: the original error is the outcome. + if (ctx.guard !== null && headsPositivelyUnbegun(ctx)) await releaseAcquiredLeases(ctx); + throw error; + } finally { + if (ctx.ordinary !== null) await ctx.ordinary.close(); + } + }); } function classifyEnteringBeforeBarrier(ctx: Context): CutoverSetActivateResult | null { @@ -543,8 +563,12 @@ async function runUnderBarrier(ctx: Context): Promise try { ctx.machineId = ctx.deps.machineId(ctx.homeDir); } catch (error) { - if (!(error instanceof MachineIdentityFileError)) throw error; - return refused(ctx, "S0", "machine-identity-unregistered", error.message).exit; + if (error instanceof MachineIdentityFileError) return refused(ctx, "S0", "machine-identity-unregistered", error.message).exit; + // A filesystem error reading the identity file is unresolvable state; anything else is not a classified condition. + if (error instanceof Error && typeof (error as NodeJS.ErrnoException).code === "string") { + return refused(ctx, "S0", "state-unresolvable", "the machine identity could not be resolved: " + error.message).exit; + } + throw error; } const records = ctx.deps.readRecords({ homeDir: ctx.homeDir, setId: ctx.setId }); if (records.state === "unresolvable") { @@ -895,7 +919,12 @@ function gate(ctx: Context, step: MigrationActivationRefusalStep): Exit | null { return refused(ctx, step, decision.reason, detail, decision.failures); } -async function identityCheck(ctx: Context, member: Member, executor: PostgreSqlQueryExecutor): Promise { +type TargetFailure = Readonly<{ reason: string; detail: string }>; + +/** A destination identity check: null when it matches, else a drift or a target failure to record on the member. */ +type IdentityOutcome = Readonly<{ kind: "drift"; detail: string }> | Readonly<{ kind: "target"; failure: TargetFailure }> | null; + +async function identityCheck(ctx: Context, member: Member, executor: PostgreSqlQueryExecutor, step: string): Promise { let live: MigrationDestinationIdentity; try { live = await ctx.deps.captureDestinationIdentity(executor, member.remoteProjectId, ctx.input.signal); @@ -910,19 +939,36 @@ async function identityCheck(ctx: Context, member: Member, executor: PostgreSqlQ ); } catch (error) { if (error instanceof Error && error.name === "MigrationVerificationDriverError") { - return "the live destination identity does not match the sealed manifest: " + error.message; + return { kind: "drift", detail: "the live destination identity does not match the sealed manifest: " + error.message }; } - throw error; + return { kind: "target", failure: targetFailure(error, step + " destination identity query") }; } if (live.sealedWitnessSha256 !== member.report!.body.destinationIdentity.sealedWitnessSha256) { - return "the live destination identity does not match the verification report"; + return { kind: "drift", detail: "the live destination identity does not match the verification report" }; } return null; } -function targetFailure(error: unknown, stage: string): Readonly<{ reason: string; detail: string }> { - if (!(error instanceof StorageOperationError)) throw error; - const attributed = classifyMigrationTargetFailure(error, stage); +/** Record an identity outcome on the member; true when the member failed. */ +function recordIdentity(member: Member, outcome: IdentityOutcome): boolean { + if (outcome === null) return false; + if (outcome.kind === "drift") fail(member, "drifted", "identity-drift", outcome.detail); + else fail(member, "unverifiable", outcome.failure.reason, outcome.failure.detail); + return true; +} + +/** + * A target failure observed by S0, S5 or S7: the StorageOperationError + * family, attributed exactly as S6 attributes the same failures. The + * PostgreSQL portable source normalizes a storage failure to + * PortableStreamError("source-unavailable") carrying the storage error as its + * cause; that wrapper alone is unwrapped. Cancellation and every other error + * are thrown. + */ +function targetFailure(error: unknown, stage: string): TargetFailure { + const failure = error instanceof PortableStreamError && error.code === "source-unavailable" ? error.cause : error; + if (!(failure instanceof StorageOperationError)) throw error; + const attributed = classifyMigrationTargetFailure(failure, stage); return { reason: attributed.status === "unhealthy" ? "target-unhealthy" : "target-unresolvable", detail: attributed.reason + ": " + attributed.detail }; } @@ -940,18 +986,26 @@ async function memberCensus(ctx: Context, member: Member): Promise { return; } try { - const drift = await identityCheck(ctx, member, session); - if (drift !== null) { fail(member, "drifted", "identity-drift", drift); return; } + if (recordIdentity(member, await identityCheck(ctx, member, session, "S7"))) return; + let scratchParent: string; + try { + scratchParent = ctx.deps.allocateScratch(ctx.homeDir); + } catch (error) { + fail(member, "unverifiable", "state-unresolvable", "the census scratch directory could not be allocated: " + (error as Error).message); + return; + } let read: Awaited>; try { read = await ctx.deps.readDestinationCensus(session, { settings: ctx.settings, expectedOwner: ctx.expectedOwner, expectedIdentity: member.identity!, - scratchParent: ctx.scratchParent, signal: ctx.input.signal, + scratchParent, signal: ctx.input.signal, }); } catch (error) { const failure = targetFailure(error, "S7 census read"); fail(member, "unverifiable", failure.reason, failure.detail); return; + } finally { + rmSync(scratchParent, { recursive: true, force: true }); } member.censusVector = ctx.deps.census.buildMigrationCensusVector(read.census); member.canonicalDelta = ctx.deps.census.buildMigrationCanonicalDelta(read.census); @@ -1143,8 +1197,7 @@ async function preSelection(ctx: Context, held: BackendMaintenanceJournal): Prom if (row !== null && row.copyBindingSha256 !== read.evidenceSha256) { return interrupted(ctx, "S9", "evidence-conflict", at(member, "the copy binding digest differs from the checkpointed one")).exit; } - const drift = await identityCheck(ctx, member, transfer); - if (drift !== null) fail(member, "drifted", "identity-drift", drift); + recordIdentity(member, await identityCheck(ctx, member, transfer, "S0")); } finally { await transfer.close(); } @@ -1173,8 +1226,7 @@ async function preSelection(ctx: Context, held: BackendMaintenanceJournal): Prom if (quiet !== null) return quiet.exit; // S5 (identity on the ordinary session) and S6 (health, once for the one target). for (const member of ctx.members) { - const drift = await identityCheck(ctx, member, ctx.ordinary!); - if (drift !== null) fail(member, "drifted", "identity-drift", drift); + recordIdentity(member, await identityCheck(ctx, member, ctx.ordinary!, "S5")); } const identified = gate(ctx, "S5"); if (identified !== null) return identified.exit; diff --git a/test/codecov-ownership/1673-cutover-set-activation-classification.yml b/test/codecov-ownership/1673-cutover-set-activation-classification.yml new file mode 100644 index 000000000..4b365fe71 --- /dev/null +++ b/test/codecov-ownership/1673-cutover-set-activation-classification.yml @@ -0,0 +1,11 @@ +references: ["#1673"] +rationale: >- + Cutover-set activation allocates census scratch per member census, classifies + destination identity and census storage failures per member, and confirms an + interrupted run as applied against the live selection, reusing the + single-generation helpers that activate-generation.ts now exports. Both files + stay in migration cutover. No production file is added, removed, moved, or + re-owned. +pins: + - ["src/migration/cutover-set-activate.ts", "unit-migration-cutover"] + - ["src/migration/activate-generation.ts", "unit-migration-cutover"] diff --git a/test/migration/cutover-set-activate-classification.test.ts b/test/migration/cutover-set-activate-classification.test.ts new file mode 100644 index 000000000..13b790760 --- /dev/null +++ b/test/migration/cutover-set-activate-classification.test.ts @@ -0,0 +1,318 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createHash } from "node:crypto"; +import { existsSync, mkdtempSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; +import { MachineIdentityFileError } from "../../src/machine-identity.js"; +import { lcmHomeDir } from "../../src/runtime-paths.js"; +import { backendPublicationCanonicalSha256, backendPublicationJournalPath } from "../../src/storage/backend-publication.js"; +import { PortableStreamError } from "../../src/storage/portable-record.js"; +import { PostgreSqlStorageOperationError } from "../../src/storage/postgresql/errors.js"; +import { activateCutoverSet, createCutoverSetActivateDependencies } from "../../src/migration/cutover-set-activate.js"; +import { MigrationBindingUnresolvableError } from "../../src/migration/generation-binding-store.js"; +import { MigrationProjectMapWitnessError } from "../../src/migration/project-map-witness.js"; +import { cleanupRoots, expectNoEffects, fixture, HASH_A, roots, TARGET_CONFIG, type Fixture } from "./cutover-set-activate-fixture.js"; +import { readCutoverSetActivationRecords } from "../../src/migration/cutover-set-activation-store.js"; + +afterEach(cleanupRoots); + +/** Cutover-set activation classification (#1673): lazy scratch (#1618), storage-error classification (#1648), evidence-derived applied (#1617). */ + +function storageError(sqlState: string, retryable: boolean): PostgreSqlStorageOperationError { + return new PostgreSqlStorageOperationError("STORAGE_OPERATION_FAILED", { domain: "factory", operation: "injected" }, sqlState, retryable); +} + +const UNHEALTHY = () => storageError("28000", false); +const UNRESOLVABLE = () => storageError("08006", true); + +function scratchDirs(homeDir: string): string[] { + return existsSync(lcmHomeDir(homeDir)) ? readdirSync(lcmHomeDir(homeDir)).filter((name) => /scratch/u.test(name)) : []; +} + +async function crashAt(f: Fixture, point: string): Promise { + await expect(f.run({ crashPoint: (p) => { if (p === point) throw new Error("crash at " + point); } })).rejects.toThrow("crash at " + point); +} + +const sha256 = (value: string | Uint8Array): string => createHash("sha256").update(value).digest("hex"); + +describe("activateCutoverSet: scratch is allocated only for a member census (#1673, #1618)", () => { + it("an early refusal creates no scratch directory and never calls the allocator", async () => { + const allocateScratch = vi.fn(() => { throw new Error("unexpected scratch allocation"); }); + const unheld = await fixture({ held: false }); + const seenUnheld: string[] = []; + const journal = unheld.deps.readMaintenanceJournal; + expect(await unheld.run({ allocateScratch, readMaintenanceJournal: (h) => { seenUnheld.push(...scratchDirs(h)); return journal(h); } })) + .toMatchObject({ status: "refused", step: "S3", reason: "maintenance-not-held" }); + expect(seenUnheld).toEqual([]); + const unregistered = await fixture(); + const seen: string[] = []; + const read = unregistered.deps.readMaintenanceJournal; + expect(await unregistered.run({ + allocateScratch, + readMaintenanceJournal: (h) => { seen.push(...scratchDirs(h)); return read(h); }, + machineId: () => { seen.push(...scratchDirs(unregistered.homeDir)); throw new MachineIdentityFileError("missing", "injected"); }, + })).toMatchObject({ status: "refused", step: "S0", reason: "machine-identity-unregistered" }); + expect(seen).toEqual([]); + expect(allocateScratch).not.toHaveBeenCalled(); + }); + + it("an uninitialized home is classified as a refusal instead of throwing from the scratch allocation", async () => { + const homeDir = mkdtempSync(join(tmpdir(), "lcm-cutover-set-empty-")); + roots.push(homeDir); + const outcome = await activateCutoverSet({ homeDir, confirmSetId: "cutover-set-1", targetConfig: TARGET_CONFIG }); + expect(outcome).toMatchObject({ status: "refused", effectBegun: false }); + expect(scratchDirs(homeDir)).toEqual([]); + }); + + it("a census scratch allocation failure makes the member unverifiable and refuses the set at S8 naming each member", async () => { + const f = await fixture(); + const outcome = await f.run({ allocateScratch: () => { throw new Error("disk full"); } }); + expect(outcome).toMatchObject({ + status: "refused", step: "S8", effectBegun: false, + members: [ + expect.objectContaining({ evidence: "unverifiable", reason: "state-unresolvable", detail: expect.stringContaining("census scratch directory could not be allocated: disk full") }), + expect.objectContaining({ evidence: "unverifiable", reason: "state-unresolvable" }), + ], + }); + expectNoEffects(f); + }); + + it("each member census gets its own scratch directory, which is removed after a census read that throws", async () => { + const f = await fixture(); + const allocated: string[] = []; + const passed: string[] = []; + const existedDuringRead: boolean[] = []; + const real = f.deps.allocateScratch; + const outcome = await f.run({ + allocateScratch: (home) => { const dir = real(home); allocated.push(dir); return dir; }, + readDestinationCensus: async (_session, census) => { + passed.push(census.scratchParent); + existedDuringRead.push(existsSync(census.scratchParent)); + throw UNHEALTHY(); + }, + }); + expect(outcome).toMatchObject({ status: "refused", step: "S8" }); + expect(allocated).toHaveLength(2); + expect(new Set(allocated).size).toBe(2); + expect(passed).toEqual(allocated); + expect(existedDuringRead).toEqual([true, true]); + for (const dir of allocated) expect(existsSync(dir)).toBe(false); + expect(scratchDirs(f.homeDir)).toEqual([]); + }); +}); + +describe("activateCutoverSet: identity and census storage failures are classified per member (#1673, #1648)", () => { + it.each([ + ["S0", 1, "S0"], + ["S5", 3, "S5"], + ["S7", 5, "S8"], + ] as const)("a StorageOperationError from the %s destination identity query refuses with the target vocabulary", async (stage, call, step) => { + for (const [make, reason] of [[UNHEALTHY, "target-unhealthy"], [UNRESOLVABLE, "target-unresolvable"]] as const) { + const f = await fixture(); + let captures = 0; + const outcome = await f.run({ captureDestinationIdentity: async () => { captures += 1; if (captures === call) throw make(); return { version: 1, ...f.deps.identity }; } }); + expect(outcome).toMatchObject({ + status: "refused", step, reason: "cutover-set-member-evidence-unverifiable", effectBegun: false, + detail: expect.stringContaining(f.members[0]!.generationId), + members: [expect.objectContaining({ evidence: "unverifiable", reason, detail: expect.stringContaining(stage + " destination identity query") }), expect.objectContaining({ evidence: null })], + }); + expectNoEffects(f); + } + }); + + it("after a checkpoint the same identity failure is a roll-forward interruption", async () => { + const f = await fixture(); + await crashAt(f, "after-checkpoint"); + let captures = 0; + expect(await f.run({ captureDestinationIdentity: async () => { captures += 1; if (captures === 1) throw UNHEALTHY(); return { version: 1, ...f.deps.identity }; } })) + .toMatchObject({ status: "interrupted", step: "S9", cause: "state-unresolvable", effectBegun: true, detail: expect.stringContaining("target-unhealthy") }); + }); + + it("a PortableStreamError(source-unavailable) wrapping a storage error from the census read refuses, per member", async () => { + for (const [make, reason] of [[UNHEALTHY, "target-unhealthy"], [UNRESOLVABLE, "target-unresolvable"]] as const) { + const f = await fixture(); + const outcome = await f.run({ readDestinationCensus: async () => { throw new PortableStreamError("source-unavailable", { cause: make() }); } }); + expect(outcome).toMatchObject({ + status: "refused", step: "S8", reason: "cutover-set-member-evidence-unverifiable", effectBegun: false, + members: [expect.objectContaining({ evidence: "unverifiable", reason, detail: expect.stringContaining("S7 census read") }), expect.objectContaining({ evidence: "unverifiable", reason })], + }); + expectNoEffects(f); + } + }); + + it("cancellation, an unwrapped source failure and a plain error still reject", async () => { + const f = await fixture(); + await expect(f.run({ readDestinationCensus: async () => { throw new PortableStreamError("aborted", { cause: UNHEALTHY() }); } })).rejects.toMatchObject({ code: "aborted" }); + await expect(f.run({ readDestinationCensus: async () => { throw new PortableStreamError("source-unavailable"); } })).rejects.toMatchObject({ code: "source-unavailable" }); + await expect(f.run({ readDestinationCensus: async () => { throw new Error("census exploded"); } })).rejects.toThrow("census exploded"); + await expect(f.run({ captureDestinationIdentity: async () => { throw new Error("capture exploded"); } })).rejects.toThrow("capture exploded"); + }); +}); + +describe("activateCutoverSet: remaining classification seams (#1673)", () => { + it("the real default allocator keeps the set scratch prefix under the LCM home", async () => { + const f = await fixture(); + const dir = createCutoverSetActivateDependencies({ homeDir: f.homeDir }).allocateScratch(f.homeDir); + try { + expect(dirname(dir)).toBe(lcmHomeDir(f.homeDir)); + expect(basename(dir)).toMatch(/^cutover-set-activation-scratch-/u); + } finally { + rmSync(dir, { recursive: true }); + } + }); + + it("an errno from the machine identity read is unresolvable state; an uncoded error rethrows", async () => { + const f = await fixture(); + expect(await f.run({ machineId: () => { throw Object.assign(new Error("permission denied"), { code: "EACCES" }); } })) + .toMatchObject({ status: "refused", step: "S0", reason: "state-unresolvable", effectBegun: false, detail: expect.stringContaining("permission denied") }); + await expect(f.run({ machineId: () => { throw new Error("uncoded"); } })).rejects.toThrow("uncoded"); + }); + + it("a member's identity storage failure does not mask a later member's checkpoint conflict", async () => { + const f = await fixture(); + const [, b] = f.members; + await crashAt(f, "after-checkpoint"); + f.deps.transferRows.set(b!.remoteProjectId, [{ run_id: "migration-copy-" + HASH_A, state: "completed" }]); + let captures = 0; + expect(await f.run({ captureDestinationIdentity: async () => { captures += 1; if (captures === 1) throw UNHEALTHY(); return { version: 1, ...f.deps.identity }; } })) + .toMatchObject({ status: "interrupted", step: "S9", cause: "evidence-conflict", effectBegun: true, detail: expect.stringContaining("copy binding digest differs") }); + }); +}); + +describe("activateCutoverSet: applied is derived from the live config and project map (#1673, #1617)", () => { + const ZERO = "0".repeat(64); + const unreadable = { readArtifact: () => null, readMaintenanceJournal: () => null } as const; + const unresolvableBinding = (onCall: () => void) => ({ + bindings: { recordMigrationSelectionBinding: () => { onCall(); throw new MigrationBindingUnresolvableError("injected"); } }, + }); + + /** A completed own publication whose project-map witness check then interrupts at S16. */ + async function afterPublication(f: Fixture, overrides: Parameters[0] = {}) { + const witness = f.deps.computeProjectMapWitness; + return f.run({ + computeProjectMapWitness: (request) => { + if (request.mode !== "post-publication") return witness(request); + throw new MigrationProjectMapWitnessError("project-map-drift", "injected drift", { drift: "entry-absent" }); + }, + ...overrides, + }); + } + + /** + * A held set at selection-completed (artifact loaded from the store) whose S13a binding write fails; the journal + * readers report `view` from that moment on, which is what the interruption's applied readback sees. + */ + async function viewed(f: Fixture, view: { maintenance?: unknown; publication?: unknown }, extra: Parameters[0] = {}) { + let lie = false; + const maintenance = f.deps.readMaintenanceJournal; + const publication = f.deps.readPublicationJournal; + return f.run({ + bindings: { ...f.deps.bindings, ...unresolvableBinding(() => { lie = true; }).bindings }, + readMaintenanceJournal: (h) => (lie ? (view.maintenance ?? null) as never : maintenance(h)), + readPublicationJournal: (h) => (lie ? (view.publication ?? null) as never : publication(h)), + ...extra, + }); + } + + async function held() { + const f = await fixture(); + await crashAt(f, "after-selection-completed"); + const live = f.deps.readSelectionDigests(f.homeDir); + return { f, live, config: live.configSha256, map: live.projectMapSha256 }; + } + + const interruptedApplied = (applied: string) => ({ status: "interrupted", step: "S13a", applied }); + + it("a completed own publication is yes while the live selection equals the published witnesses and unknown otherwise", async () => { + const f = await fixture(); + expect(await afterPublication(f)).toMatchObject({ status: "interrupted", step: "S16", applied: "yes" }); + const g = await fixture(); + expect(await afterPublication(g, { readSelectionDigests: () => ({ configSha256: ZERO, projectMapSha256: ZERO }) })).toMatchObject({ status: "interrupted", step: "S16", applied: "unknown" }); + const h = await fixture(); + expect(await afterPublication(h, { readSelectionDigests: () => { throw new Error("readback failed"); } })).toMatchObject({ status: "interrupted", step: "S16", applied: "unknown" }); + }); + + it("a yes-phase own journal needs both published witnesses to equal the live selection", async () => { + const { f, config, map } = await held(); + for (const phase of ["config-published", "releasing", "released", "completed"]) { + const own = (extra: object = {}) => ({ publicationId: f.setId, phase, publishedConfigSha256: config, publishedProjectMapSha256: map, ...extra }); + expect(await viewed(f, { publication: own() })).toMatchObject(interruptedApplied("yes")); + expect(await viewed(f, { publication: own({ publishedConfigSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + expect(await viewed(f, { publication: own({ publishedProjectMapSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + expect(await viewed(f, { publication: own({ publishedConfigSha256: null }) })).toMatchObject(interruptedApplied("unknown")); + expect(await viewed(f, { publication: own({ publishedProjectMapSha256: null }) })).toMatchObject(interruptedApplied("unknown")); + } + }); + + it("an in-flight own phase is unknown even when the live selection equals the source", async () => { + const { f, config, map } = await held(); + for (const phase of ["map-publishing", "map-published", "config-publishing"]) { + expect(await viewed(f, { publication: { publicationId: f.setId, phase, expectedConfigSha256: config, expectedProjectMapSha256: map } })).toMatchObject(interruptedApplied("unknown")); + } + }); + + it("a pre-publication or abort phase of the own journal is no only while the live selection equals the expected witnesses", async () => { + const { f, config, map } = await held(); + for (const phase of ["preparing", "prepared", "acquiring", "guarded", "aborting", "config-restoring", "map-restoring", "abort-releasing", "aborted"]) { + const own = (extra: object = {}) => ({ publicationId: f.setId, phase, expectedConfigSha256: config, expectedProjectMapSha256: map, ...extra }); + expect(await viewed(f, { publication: own() })).toMatchObject(interruptedApplied("no")); + expect(await viewed(f, { publication: own({ expectedConfigSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + expect(await viewed(f, { publication: own({ expectedProjectMapSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + } + }); + + it("an own journal whose readback fails is unknown", async () => { + const { f, config, map } = await held(); + const own = { publicationId: f.setId, phase: "completed", publishedConfigSha256: config, publishedProjectMapSha256: map }; + expect(await viewed(f, { publication: own }, { readSelectionDigests: () => { throw new Error("readback failed"); } })).toMatchObject(interruptedApplied("unknown")); + let lie = false; + const publication = f.deps.readPublicationJournal; + expect(await f.run({ + bindings: { ...f.deps.bindings, ...unresolvableBinding(() => { lie = true; }).bindings }, + readPublicationJournal: (h) => { if (lie) throw new Error("journal read failed"); return publication(h); }, + readMaintenanceJournal: (h) => (lie ? null : f.deps.readMaintenanceJournal(h)), + })).toMatchObject(interruptedApplied("unknown")); + }); + + it("a held sqlite window is no only while the live selection equals the artifact's source witness, and a window from another backend is unknown", async () => { + const { f } = await held(); + const records = readCutoverSetActivationRecords({ homeDir: f.homeDir, setId: f.setId }); + if (records.state !== "resolved") throw new Error("records not resolved"); + const source = f.deps.readArtifact({ homeDir: f.homeDir, identityDigest: records.checkpoint!.identityDigest })!.source; + const live = f.deps.readSelectionDigests(f.homeDir); + expect(source.config.presence).toBe("present"); + expect(source.projectMap.presence).toBe("present"); + expect(live.configSha256).toBe(sha256((source.config as { content: Uint8Array }).content)); + expect(live.projectMapSha256).toBe(backendPublicationCanonicalSha256(JSON.parse(Buffer.from((source.projectMap as { content: Uint8Array }).content).toString("utf8")))); + const resolved = { bindings: { ...f.deps.bindings, ...unresolvableBinding(() => undefined).bindings } }; + expect(await f.run(resolved)).toMatchObject(interruptedApplied("no")); + expect(await f.run({ ...resolved, readSelectionDigests: () => ({ ...live, configSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + expect(await f.run({ ...resolved, readSelectionDigests: () => ({ ...live, projectMapSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + expect(await f.run({ ...resolved, readSelectionDigests: () => { throw new Error("readback failed"); } })).toMatchObject(interruptedApplied("unknown")); + expect(await viewed(f, { maintenance: { ...f.journal()!, sourceBackend: "postgresql" } })).toMatchObject(interruptedApplied("unknown")); + }); + + it("an absent or foreign publication journal is no only while the live selection equals the artifact's source witness", async () => { + const { f, live } = await held(); + for (const publication of [null, { publicationId: "other", phase: "completed" }]) { + expect(await viewed(f, { publication })).toMatchObject(interruptedApplied("no")); + expect(await viewed(f, { publication }, { readSelectionDigests: () => ({ ...live, projectMapSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + } + }); + + it("an empty slot is no only while the live selection equals the artifact's source witness", async () => { + const f = await fixture(); + await crashAt(f, "after-selection-completed"); + rmSync(backendPublicationJournalPath(f.homeDir)); + const binding = { bindings: { ...f.deps.bindings, ...unresolvableBinding(() => undefined).bindings } }; + expect(await f.run(binding)).toMatchObject(interruptedApplied("no")); + expect(await f.run({ ...binding, readSelectionDigests: () => ({ configSha256: ZERO, projectMapSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + }); + + it("an absent or foreign publication journal without a source witness is unknown", async () => { + const f = await fixture(); + await crashAt(f, "after-selection-completed"); + expect(await f.run({ ...unreadable, readPublicationJournal: () => null })).toMatchObject({ status: "interrupted", applied: "unknown" }); + expect(await f.run({ ...unreadable, readPublicationJournal: () => ({ publicationId: "other", phase: "completed" }) as never })).toMatchObject({ status: "interrupted", applied: "unknown" }); + }); +}); diff --git a/test/migration/cutover-set-activate-faults.test.ts b/test/migration/cutover-set-activate-faults.test.ts index af16ad819..0bc444242 100644 --- a/test/migration/cutover-set-activate-faults.test.ts +++ b/test/migration/cutover-set-activate-faults.test.ts @@ -102,7 +102,7 @@ describe("activateCutoverSet faults: entry, records and journals", () => { await crashAt(f, "after-checkpoint"); reads = 0; expect(await f.run({ readMaintenanceJournal: (homeDir) => { reads += 1; if (reads === 2) throw new Error("injected under barrier"); return journal(homeDir); } })) - .toMatchObject({ status: "interrupted", step: "S12", cause: "journal-unexpected", applied: "no" }); + .toMatchObject({ status: "interrupted", step: "S12", cause: "journal-unexpected", applied: "unknown" }); }); it("classifies applied from the journals when an interruption is reported", async () => { @@ -113,9 +113,9 @@ describe("activateCutoverSet faults: entry, records and journals", () => { // The journal read inside classifyApplied throws: applied is unknown. expect(await f.run({ readMaintenanceJournal: (homeDir) => { reads += 1; if (reads > 2) throw new Error("later read"); return journal(homeDir); }, readArtifact: () => null })) .toMatchObject({ status: "interrupted", applied: "unknown" }); - // A live publication of another id, and an in-flight own publication. + // A live publication of another id with no source witness in the context, and an in-flight own publication. expect(await f.run({ readArtifact: () => null, readMaintenanceJournal: () => null, readPublicationJournal: () => ({ publicationId: "other", phase: "completed" }) as never })) - .toMatchObject({ status: "interrupted", applied: "no" }); + .toMatchObject({ status: "interrupted", applied: "unknown" }); expect(await f.run({ readArtifact: () => null, readMaintenanceJournal: () => null, readPublicationJournal: () => ({ publicationId: f.setId, phase: "map-published" }) as never })) .toMatchObject({ status: "interrupted", applied: "unknown" }); }); From 15db01ed259d021ed05023fe4f6851bec3d6da11 Mon Sep 17 00:00:00 2001 From: Bernardo Donadio Date: Sun, 25 Oct 2026 00:48:49 -0300 Subject: [PATCH 2/2] test: tighten empty-home activation assertions Review of 8ec42d45 asked for the uninitialized-home test to pin the classification it relies on. It now runs the real default dependencies with a pass-through spy on allocateScratch and asserts a refusal at S0 with machine-identity-unregistered, effectBegun false, no allocator call and no scratch directory. The set activation doc said an early refusal leaves nothing on disk, but the append barrier still creates the empty LCM home. It now says only that an early refusal allocates no census scratch directory. Signed-off-by: Bernardo Donadio --- docs/migration-cutover-set.md | 6 +++--- test/migration/cutover-set-activate-classification.test.ts | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/migration-cutover-set.md b/docs/migration-cutover-set.md index 2659a5219..4ed46b80b 100644 --- a/docs/migration-cutover-set.md +++ b/docs/migration-cutover-set.md @@ -294,9 +294,9 @@ The set activation follows the single-generation rules for scratch space, target failures and the `applied` answer of an interruption (see [Migration activation](migration-cutover.md#migration-activation)). A member's census scratch directory is allocated only when that member's census is read -and is removed afterwards, so an early refusal leaves nothing on disk. A -storage failure of a member's destination identity query or census read is -attributed to that member as `target-unhealthy` or `target-unresolvable`, and +and is removed afterwards, so an early refusal allocates no census scratch +directory. A storage failure of a member's destination identity query or census +read is attributed to that member as `target-unhealthy` or `target-unresolvable`, and the set is refused naming it. An interrupted outcome reports `applied` as `yes` or `no` only when the live configuration and project map agree with the publication journal or the set's pre-publication source, otherwise `unknown`. diff --git a/test/migration/cutover-set-activate-classification.test.ts b/test/migration/cutover-set-activate-classification.test.ts index 13b790760..8acca02a1 100644 --- a/test/migration/cutover-set-activate-classification.test.ts +++ b/test/migration/cutover-set-activate-classification.test.ts @@ -59,8 +59,11 @@ describe("activateCutoverSet: scratch is allocated only for a member census (#16 it("an uninitialized home is classified as a refusal instead of throwing from the scratch allocation", async () => { const homeDir = mkdtempSync(join(tmpdir(), "lcm-cutover-set-empty-")); roots.push(homeDir); - const outcome = await activateCutoverSet({ homeDir, confirmSetId: "cutover-set-1", targetConfig: TARGET_CONFIG }); - expect(outcome).toMatchObject({ status: "refused", effectBegun: false }); + const defaults = createCutoverSetActivateDependencies({ homeDir }); + const allocateScratch = vi.fn(defaults.allocateScratch); + const outcome = await activateCutoverSet({ homeDir, confirmSetId: "cutover-set-1", targetConfig: TARGET_CONFIG }, { ...defaults, allocateScratch }); + expect(outcome).toMatchObject({ status: "refused", step: "S0", reason: "machine-identity-unregistered", effectBegun: false }); + expect(allocateScratch).not.toHaveBeenCalled(); expect(scratchDirs(homeDir)).toEqual([]); });