diff --git a/.changeset/1673-cutover-set-activation-classification.md b/.changeset/1673-cutover-set-activation-classification.md new file mode 100644 index 000000000..e9066ae40 --- /dev/null +++ b/.changeset/1673-cutover-set-activation-classification.md @@ -0,0 +1,9 @@ +--- +"@donadiosolutions/lcm": patch +--- + +Cutover-set activation now allocates census scratch only for a member census, +reports a storage failure of a destination identity query or census read as a +target refusal naming the member, and reports an interrupted run as applied +only when the live configuration and project map agree with the publication +journal or the set's pre-publication source. diff --git a/docs/migration-cutover-set.md b/docs/migration-cutover-set.md index a1231dcbc..4ed46b80b 100644 --- a/docs/migration-cutover-set.md +++ b/docs/migration-cutover-set.md @@ -290,6 +290,17 @@ call) and the coverage check may have reconciled the project map. No member's evidence is ever inferred from another's, and no time, count or heuristic decides eligibility. +The set activation follows the single-generation rules for scratch space, +target failures and the `applied` answer of an interruption (see +[Migration activation](migration-cutover.md#migration-activation)). A member's +census scratch directory is allocated only when that member's census is read +and is removed afterwards, so an early refusal allocates no census scratch +directory. A storage failure of a member's destination identity query or census +read is attributed to that member as `target-unhealthy` or `target-unresolvable`, and +the set is refused naming it. An interrupted outcome reports `applied` as `yes` +or `no` only when the live configuration and project map agree with the +publication journal or the set's pre-publication source, otherwise `unknown`. + ### One checkpoint, member effects, one selection, one publication Once every member passes, the publication material is composed once for all diff --git a/src/migration/activate-generation.ts b/src/migration/activate-generation.ts index 0bc893e95..eddc7ee7d 100644 --- a/src/migration/activate-generation.ts +++ b/src/migration/activate-generation.ts @@ -800,14 +800,14 @@ function projectBinding(ctx: Context): ProjectBinding { /** The own journal records the config as published: yes needs the live selection to match it. */ const APPLIED_YES_PHASES: readonly BackendPublicationJournal["phase"][] = ["config-published", "releasing", "released", "completed"]; /** The write may be in flight: never proven either way, even when the live selection equals the source. */ -const APPLIED_IN_FLIGHT_PHASES: readonly BackendPublicationJournal["phase"][] = ["map-publishing", "map-published", "config-publishing"]; +export const APPLIED_IN_FLIGHT_PHASES: readonly BackendPublicationJournal["phase"][] = ["map-publishing", "map-published", "config-publishing"]; -function digestsMatch(live: MigrationActivationSelectionDigests, configSha256: string, projectMapSha256: string): boolean { +export function digestsMatch(live: MigrationActivationSelectionDigests, configSha256: string, projectMapSha256: string): boolean { return live.configSha256 === configSha256 && live.projectMapSha256 === projectMapSha256; } /** The digest of a recovery file under the journal convention: raw bytes for the config, canonical JSON for the map, {} when absent. */ -function recoveryFileDigests(config: BackendPublicationRecoveryFile, projectMap: BackendPublicationRecoveryFile): MigrationActivationSelectionDigests { +export function recoveryFileDigests(config: BackendPublicationRecoveryFile, projectMap: BackendPublicationRecoveryFile): MigrationActivationSelectionDigests { return { configSha256: createHash("sha256").update(config.presence === "present" ? config.content : "{}").digest("hex"), projectMapSha256: backendPublicationCanonicalSha256( diff --git a/src/migration/cutover-set-activate.ts b/src/migration/cutover-set-activate.ts index d6ab6bf7d..13edc0c6a 100644 --- a/src/migration/cutover-set-activate.ts +++ b/src/migration/cutover-set-activate.ts @@ -21,6 +21,7 @@ import { import type { ComposeBackendPublicationMaterialResult } from "../storage/backend-publication-material.js"; import type { StorageIdentityContext } from "../storage/contracts.js"; import { StorageOperationError } from "../storage/errors.js"; +import { PortableStreamError } from "../storage/portable-record.js"; import type { PostgreSqlConnectionSettings, PostgreSqlQueryExecutor } from "../storage/postgresql/contracts.js"; import { connectionSettings } from "../storage/postgresql/factory.js"; import { @@ -28,7 +29,9 @@ import { POSTGRESQL_BACKEND_PUBLICATION_RESOURCE_TYPE, } from "../storage/postgresql/publication-guard.js"; import { + APPLIED_IN_FLIGHT_PHASES, createMigrationActivationDependencies, + digestsMatch, MigrationActivationError, type MigrationActivationDependencies, type MigrationActivationInterruptionCause, @@ -37,6 +40,7 @@ import { type MigrationActivationRefusalReason, type MigrationActivationRefusalStep, type MigrationActivationSnapshot, + recoveryFileDigests, type MigrationActivationTriState, } from "./activate-generation.js"; import { @@ -284,6 +288,7 @@ export type CutoverSetActivateDependencies = MigrationActivationDependencies & R export function createCutoverSetActivateDependencies(input: Readonly<{ homeDir: string }>): CutoverSetActivateDependencies { return { ...createMigrationActivationDependencies(input), + allocateScratch: (home) => mkdtempSync(join(lcmHomeDir(home), "cutover-set-activation-scratch-")), resolveAuthorities: resolveCutoverSetMemberAuthorities, readCanonicalPath: readProjectMapCanonicalPath, resolveExpectedIdentity: resolveMigrationExpectedIdentity, @@ -354,7 +359,6 @@ type Context = { readonly coordinates: NonSecretPostgreSqlTarget; readonly maxProbeOrdinal: number; readonly now: () => Date; - readonly scratchParent: string; machineId: string; token: BackendPublicationLockToken; manifest: BackendCutoverSetManifest | null; @@ -422,20 +426,41 @@ function refused( return { exit: { status: "refused", setId: ctx.setId, step, reason, detail, failures, members: memberResults(ctx), effectBegun: false } }; } +/** The own journal records the config as published: yes needs the live selection to match it. */ const APPLIED_YES_PHASES: readonly BackendPublicationJournal["phase"][] = ["config-published", "releasing", "released", "completed"]; +/** "no" only when the live selection still equals the set's pre-publication source witness; otherwise "unknown". */ +function classifyAgainstSource(ctx: Context): MigrationActivationTriState { + // The witness enters the context at compose or at the stored artifact load; + // an interruption before either has no source proof. + const source = ctx.material?.material.source ?? ctx.artifact?.source; + if (source === undefined) return "unknown"; + const witness = recoveryFileDigests(source.config, source.projectMap); + return digestsMatch(ctx.deps.readSelectionDigests(ctx.homeDir), witness.configSha256, witness.projectMapSha256) ? "no" : "unknown"; +} + +/** + * applied is evidence-derived: the journal phase says what was attempted, and + * an independent readback of the live config and project map must agree + * before it is reported (the single-generation rules of activate-generation.ts). + * Everything else, including any readback failure, is unknown. Never throws. + */ function classifyApplied(ctx: Context): MigrationActivationTriState { - let journal: BackendPublicationJournal | null; try { const maintenance = ctx.deps.readMaintenanceJournal(ctx.homeDir); - if (maintenance !== null) return maintenance.sourceBackend === "sqlite" ? "no" : "unknown"; - journal = ctx.deps.readPublicationJournal(ctx.homeDir); + if (maintenance !== null) return maintenance.sourceBackend === "sqlite" ? classifyAgainstSource(ctx) : "unknown"; + const journal = ctx.deps.readPublicationJournal(ctx.homeDir); + if (journal === null || journal.publicationId !== ctx.setId) return classifyAgainstSource(ctx); + if (APPLIED_IN_FLIGHT_PHASES.includes(journal.phase)) return "unknown"; + const live = ctx.deps.readSelectionDigests(ctx.homeDir); + if (APPLIED_YES_PHASES.includes(journal.phase)) { + const { publishedConfigSha256, publishedProjectMapSha256 } = journal; + return publishedConfigSha256 !== null && publishedProjectMapSha256 !== null && digestsMatch(live, publishedConfigSha256, publishedProjectMapSha256) ? "yes" : "unknown"; + } + return digestsMatch(live, journal.expectedConfigSha256, journal.expectedProjectMapSha256) ? "no" : "unknown"; } catch { return "unknown"; } - if (journal === null || journal.publicationId !== ctx.setId) return "no"; - if (APPLIED_YES_PHASES.includes(journal.phase)) return "yes"; - return "unknown"; } function interrupted( @@ -492,37 +517,32 @@ export async function activateCutoverSet( // The coordinates helper refuses only through its typed MigrationIdentityError. throw new MigrationActivationError("invalid-input", "cutover-set activation target is invalid: " + (error as Error).message); } - const scratchParent = mkdtempSync(join(lcmHomeDir(input.homeDir), "cutover-set-activation-scratch-")); const ctx: Context = { input, deps, homeDir: input.homeDir, setId: input.confirmSetId, settings: connectionSettings(input.targetConfig), expectedOwner: input.targetConfig.postgresql.migrationRole, coordinates, maxProbeOrdinal: input.maxProbeOrdinal ?? DEFAULT_MAX_PROBE_ORDINAL, - now: input.now ?? (() => new Date()), scratchParent, + now: input.now ?? (() => new Date()), machineId: "", token: {} as BackendPublicationLockToken, manifest: null, members: [], journal: null, records: null, checkpoint: null, selection: null, artifact: null, material: null, identityDigest: null, ordinary: null, guard: null, coordinator: null, effectBegun: false, resumedFrom: "fresh", }; - try { - // The append barrier itself refuses a maintenance-entering journal, so that - // row of the routing table is classified before the barrier (as the - // preparation classifies its entry); every other row is read again under it. - const entering = classifyEnteringBeforeBarrier(ctx); - if (entering !== null) return entering; - return await deps.withAppendBarrier(input.homeDir, async (token) => { - ctx.token = token; - try { - return await withPreEffectLeasesReleased(ctx, await runUnderBarrier(ctx)); - } catch (error) { - // Best effort, only when readback proves no member effect began: the original error is the outcome. - if (ctx.guard !== null && headsPositivelyUnbegun(ctx)) await releaseAcquiredLeases(ctx); - throw error; - } finally { - if (ctx.ordinary !== null) await ctx.ordinary.close(); - } - }); - } finally { - rmSync(scratchParent, { recursive: true, force: true }); - } + // The append barrier itself refuses a maintenance-entering journal, so that + // row of the routing table is classified before the barrier (as the + // preparation classifies its entry); every other row is read again under it. + const entering = classifyEnteringBeforeBarrier(ctx); + if (entering !== null) return entering; + return await deps.withAppendBarrier(input.homeDir, async (token) => { + ctx.token = token; + try { + return await withPreEffectLeasesReleased(ctx, await runUnderBarrier(ctx)); + } catch (error) { + // Best effort, only when readback proves no member effect began: the original error is the outcome. + if (ctx.guard !== null && headsPositivelyUnbegun(ctx)) await releaseAcquiredLeases(ctx); + throw error; + } finally { + if (ctx.ordinary !== null) await ctx.ordinary.close(); + } + }); } function classifyEnteringBeforeBarrier(ctx: Context): CutoverSetActivateResult | null { @@ -543,8 +563,12 @@ async function runUnderBarrier(ctx: Context): Promise try { ctx.machineId = ctx.deps.machineId(ctx.homeDir); } catch (error) { - if (!(error instanceof MachineIdentityFileError)) throw error; - return refused(ctx, "S0", "machine-identity-unregistered", error.message).exit; + if (error instanceof MachineIdentityFileError) return refused(ctx, "S0", "machine-identity-unregistered", error.message).exit; + // A filesystem error reading the identity file is unresolvable state; anything else is not a classified condition. + if (error instanceof Error && typeof (error as NodeJS.ErrnoException).code === "string") { + return refused(ctx, "S0", "state-unresolvable", "the machine identity could not be resolved: " + error.message).exit; + } + throw error; } const records = ctx.deps.readRecords({ homeDir: ctx.homeDir, setId: ctx.setId }); if (records.state === "unresolvable") { @@ -895,7 +919,12 @@ function gate(ctx: Context, step: MigrationActivationRefusalStep): Exit | null { return refused(ctx, step, decision.reason, detail, decision.failures); } -async function identityCheck(ctx: Context, member: Member, executor: PostgreSqlQueryExecutor): Promise { +type TargetFailure = Readonly<{ reason: string; detail: string }>; + +/** A destination identity check: null when it matches, else a drift or a target failure to record on the member. */ +type IdentityOutcome = Readonly<{ kind: "drift"; detail: string }> | Readonly<{ kind: "target"; failure: TargetFailure }> | null; + +async function identityCheck(ctx: Context, member: Member, executor: PostgreSqlQueryExecutor, step: string): Promise { let live: MigrationDestinationIdentity; try { live = await ctx.deps.captureDestinationIdentity(executor, member.remoteProjectId, ctx.input.signal); @@ -910,19 +939,36 @@ async function identityCheck(ctx: Context, member: Member, executor: PostgreSqlQ ); } catch (error) { if (error instanceof Error && error.name === "MigrationVerificationDriverError") { - return "the live destination identity does not match the sealed manifest: " + error.message; + return { kind: "drift", detail: "the live destination identity does not match the sealed manifest: " + error.message }; } - throw error; + return { kind: "target", failure: targetFailure(error, step + " destination identity query") }; } if (live.sealedWitnessSha256 !== member.report!.body.destinationIdentity.sealedWitnessSha256) { - return "the live destination identity does not match the verification report"; + return { kind: "drift", detail: "the live destination identity does not match the verification report" }; } return null; } -function targetFailure(error: unknown, stage: string): Readonly<{ reason: string; detail: string }> { - if (!(error instanceof StorageOperationError)) throw error; - const attributed = classifyMigrationTargetFailure(error, stage); +/** Record an identity outcome on the member; true when the member failed. */ +function recordIdentity(member: Member, outcome: IdentityOutcome): boolean { + if (outcome === null) return false; + if (outcome.kind === "drift") fail(member, "drifted", "identity-drift", outcome.detail); + else fail(member, "unverifiable", outcome.failure.reason, outcome.failure.detail); + return true; +} + +/** + * A target failure observed by S0, S5 or S7: the StorageOperationError + * family, attributed exactly as S6 attributes the same failures. The + * PostgreSQL portable source normalizes a storage failure to + * PortableStreamError("source-unavailable") carrying the storage error as its + * cause; that wrapper alone is unwrapped. Cancellation and every other error + * are thrown. + */ +function targetFailure(error: unknown, stage: string): TargetFailure { + const failure = error instanceof PortableStreamError && error.code === "source-unavailable" ? error.cause : error; + if (!(failure instanceof StorageOperationError)) throw error; + const attributed = classifyMigrationTargetFailure(failure, stage); return { reason: attributed.status === "unhealthy" ? "target-unhealthy" : "target-unresolvable", detail: attributed.reason + ": " + attributed.detail }; } @@ -940,18 +986,26 @@ async function memberCensus(ctx: Context, member: Member): Promise { return; } try { - const drift = await identityCheck(ctx, member, session); - if (drift !== null) { fail(member, "drifted", "identity-drift", drift); return; } + if (recordIdentity(member, await identityCheck(ctx, member, session, "S7"))) return; + let scratchParent: string; + try { + scratchParent = ctx.deps.allocateScratch(ctx.homeDir); + } catch (error) { + fail(member, "unverifiable", "state-unresolvable", "the census scratch directory could not be allocated: " + (error as Error).message); + return; + } let read: Awaited>; try { read = await ctx.deps.readDestinationCensus(session, { settings: ctx.settings, expectedOwner: ctx.expectedOwner, expectedIdentity: member.identity!, - scratchParent: ctx.scratchParent, signal: ctx.input.signal, + scratchParent, signal: ctx.input.signal, }); } catch (error) { const failure = targetFailure(error, "S7 census read"); fail(member, "unverifiable", failure.reason, failure.detail); return; + } finally { + rmSync(scratchParent, { recursive: true, force: true }); } member.censusVector = ctx.deps.census.buildMigrationCensusVector(read.census); member.canonicalDelta = ctx.deps.census.buildMigrationCanonicalDelta(read.census); @@ -1143,8 +1197,7 @@ async function preSelection(ctx: Context, held: BackendMaintenanceJournal): Prom if (row !== null && row.copyBindingSha256 !== read.evidenceSha256) { return interrupted(ctx, "S9", "evidence-conflict", at(member, "the copy binding digest differs from the checkpointed one")).exit; } - const drift = await identityCheck(ctx, member, transfer); - if (drift !== null) fail(member, "drifted", "identity-drift", drift); + recordIdentity(member, await identityCheck(ctx, member, transfer, "S0")); } finally { await transfer.close(); } @@ -1173,8 +1226,7 @@ async function preSelection(ctx: Context, held: BackendMaintenanceJournal): Prom if (quiet !== null) return quiet.exit; // S5 (identity on the ordinary session) and S6 (health, once for the one target). for (const member of ctx.members) { - const drift = await identityCheck(ctx, member, ctx.ordinary!); - if (drift !== null) fail(member, "drifted", "identity-drift", drift); + recordIdentity(member, await identityCheck(ctx, member, ctx.ordinary!, "S5")); } const identified = gate(ctx, "S5"); if (identified !== null) return identified.exit; diff --git a/test/codecov-ownership/1673-cutover-set-activation-classification.yml b/test/codecov-ownership/1673-cutover-set-activation-classification.yml new file mode 100644 index 000000000..4b365fe71 --- /dev/null +++ b/test/codecov-ownership/1673-cutover-set-activation-classification.yml @@ -0,0 +1,11 @@ +references: ["#1673"] +rationale: >- + Cutover-set activation allocates census scratch per member census, classifies + destination identity and census storage failures per member, and confirms an + interrupted run as applied against the live selection, reusing the + single-generation helpers that activate-generation.ts now exports. Both files + stay in migration cutover. No production file is added, removed, moved, or + re-owned. +pins: + - ["src/migration/cutover-set-activate.ts", "unit-migration-cutover"] + - ["src/migration/activate-generation.ts", "unit-migration-cutover"] diff --git a/test/migration/cutover-set-activate-classification.test.ts b/test/migration/cutover-set-activate-classification.test.ts new file mode 100644 index 000000000..8acca02a1 --- /dev/null +++ b/test/migration/cutover-set-activate-classification.test.ts @@ -0,0 +1,321 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createHash } from "node:crypto"; +import { existsSync, mkdtempSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; +import { MachineIdentityFileError } from "../../src/machine-identity.js"; +import { lcmHomeDir } from "../../src/runtime-paths.js"; +import { backendPublicationCanonicalSha256, backendPublicationJournalPath } from "../../src/storage/backend-publication.js"; +import { PortableStreamError } from "../../src/storage/portable-record.js"; +import { PostgreSqlStorageOperationError } from "../../src/storage/postgresql/errors.js"; +import { activateCutoverSet, createCutoverSetActivateDependencies } from "../../src/migration/cutover-set-activate.js"; +import { MigrationBindingUnresolvableError } from "../../src/migration/generation-binding-store.js"; +import { MigrationProjectMapWitnessError } from "../../src/migration/project-map-witness.js"; +import { cleanupRoots, expectNoEffects, fixture, HASH_A, roots, TARGET_CONFIG, type Fixture } from "./cutover-set-activate-fixture.js"; +import { readCutoverSetActivationRecords } from "../../src/migration/cutover-set-activation-store.js"; + +afterEach(cleanupRoots); + +/** Cutover-set activation classification (#1673): lazy scratch (#1618), storage-error classification (#1648), evidence-derived applied (#1617). */ + +function storageError(sqlState: string, retryable: boolean): PostgreSqlStorageOperationError { + return new PostgreSqlStorageOperationError("STORAGE_OPERATION_FAILED", { domain: "factory", operation: "injected" }, sqlState, retryable); +} + +const UNHEALTHY = () => storageError("28000", false); +const UNRESOLVABLE = () => storageError("08006", true); + +function scratchDirs(homeDir: string): string[] { + return existsSync(lcmHomeDir(homeDir)) ? readdirSync(lcmHomeDir(homeDir)).filter((name) => /scratch/u.test(name)) : []; +} + +async function crashAt(f: Fixture, point: string): Promise { + await expect(f.run({ crashPoint: (p) => { if (p === point) throw new Error("crash at " + point); } })).rejects.toThrow("crash at " + point); +} + +const sha256 = (value: string | Uint8Array): string => createHash("sha256").update(value).digest("hex"); + +describe("activateCutoverSet: scratch is allocated only for a member census (#1673, #1618)", () => { + it("an early refusal creates no scratch directory and never calls the allocator", async () => { + const allocateScratch = vi.fn(() => { throw new Error("unexpected scratch allocation"); }); + const unheld = await fixture({ held: false }); + const seenUnheld: string[] = []; + const journal = unheld.deps.readMaintenanceJournal; + expect(await unheld.run({ allocateScratch, readMaintenanceJournal: (h) => { seenUnheld.push(...scratchDirs(h)); return journal(h); } })) + .toMatchObject({ status: "refused", step: "S3", reason: "maintenance-not-held" }); + expect(seenUnheld).toEqual([]); + const unregistered = await fixture(); + const seen: string[] = []; + const read = unregistered.deps.readMaintenanceJournal; + expect(await unregistered.run({ + allocateScratch, + readMaintenanceJournal: (h) => { seen.push(...scratchDirs(h)); return read(h); }, + machineId: () => { seen.push(...scratchDirs(unregistered.homeDir)); throw new MachineIdentityFileError("missing", "injected"); }, + })).toMatchObject({ status: "refused", step: "S0", reason: "machine-identity-unregistered" }); + expect(seen).toEqual([]); + expect(allocateScratch).not.toHaveBeenCalled(); + }); + + it("an uninitialized home is classified as a refusal instead of throwing from the scratch allocation", async () => { + const homeDir = mkdtempSync(join(tmpdir(), "lcm-cutover-set-empty-")); + roots.push(homeDir); + const defaults = createCutoverSetActivateDependencies({ homeDir }); + const allocateScratch = vi.fn(defaults.allocateScratch); + const outcome = await activateCutoverSet({ homeDir, confirmSetId: "cutover-set-1", targetConfig: TARGET_CONFIG }, { ...defaults, allocateScratch }); + expect(outcome).toMatchObject({ status: "refused", step: "S0", reason: "machine-identity-unregistered", effectBegun: false }); + expect(allocateScratch).not.toHaveBeenCalled(); + expect(scratchDirs(homeDir)).toEqual([]); + }); + + it("a census scratch allocation failure makes the member unverifiable and refuses the set at S8 naming each member", async () => { + const f = await fixture(); + const outcome = await f.run({ allocateScratch: () => { throw new Error("disk full"); } }); + expect(outcome).toMatchObject({ + status: "refused", step: "S8", effectBegun: false, + members: [ + expect.objectContaining({ evidence: "unverifiable", reason: "state-unresolvable", detail: expect.stringContaining("census scratch directory could not be allocated: disk full") }), + expect.objectContaining({ evidence: "unverifiable", reason: "state-unresolvable" }), + ], + }); + expectNoEffects(f); + }); + + it("each member census gets its own scratch directory, which is removed after a census read that throws", async () => { + const f = await fixture(); + const allocated: string[] = []; + const passed: string[] = []; + const existedDuringRead: boolean[] = []; + const real = f.deps.allocateScratch; + const outcome = await f.run({ + allocateScratch: (home) => { const dir = real(home); allocated.push(dir); return dir; }, + readDestinationCensus: async (_session, census) => { + passed.push(census.scratchParent); + existedDuringRead.push(existsSync(census.scratchParent)); + throw UNHEALTHY(); + }, + }); + expect(outcome).toMatchObject({ status: "refused", step: "S8" }); + expect(allocated).toHaveLength(2); + expect(new Set(allocated).size).toBe(2); + expect(passed).toEqual(allocated); + expect(existedDuringRead).toEqual([true, true]); + for (const dir of allocated) expect(existsSync(dir)).toBe(false); + expect(scratchDirs(f.homeDir)).toEqual([]); + }); +}); + +describe("activateCutoverSet: identity and census storage failures are classified per member (#1673, #1648)", () => { + it.each([ + ["S0", 1, "S0"], + ["S5", 3, "S5"], + ["S7", 5, "S8"], + ] as const)("a StorageOperationError from the %s destination identity query refuses with the target vocabulary", async (stage, call, step) => { + for (const [make, reason] of [[UNHEALTHY, "target-unhealthy"], [UNRESOLVABLE, "target-unresolvable"]] as const) { + const f = await fixture(); + let captures = 0; + const outcome = await f.run({ captureDestinationIdentity: async () => { captures += 1; if (captures === call) throw make(); return { version: 1, ...f.deps.identity }; } }); + expect(outcome).toMatchObject({ + status: "refused", step, reason: "cutover-set-member-evidence-unverifiable", effectBegun: false, + detail: expect.stringContaining(f.members[0]!.generationId), + members: [expect.objectContaining({ evidence: "unverifiable", reason, detail: expect.stringContaining(stage + " destination identity query") }), expect.objectContaining({ evidence: null })], + }); + expectNoEffects(f); + } + }); + + it("after a checkpoint the same identity failure is a roll-forward interruption", async () => { + const f = await fixture(); + await crashAt(f, "after-checkpoint"); + let captures = 0; + expect(await f.run({ captureDestinationIdentity: async () => { captures += 1; if (captures === 1) throw UNHEALTHY(); return { version: 1, ...f.deps.identity }; } })) + .toMatchObject({ status: "interrupted", step: "S9", cause: "state-unresolvable", effectBegun: true, detail: expect.stringContaining("target-unhealthy") }); + }); + + it("a PortableStreamError(source-unavailable) wrapping a storage error from the census read refuses, per member", async () => { + for (const [make, reason] of [[UNHEALTHY, "target-unhealthy"], [UNRESOLVABLE, "target-unresolvable"]] as const) { + const f = await fixture(); + const outcome = await f.run({ readDestinationCensus: async () => { throw new PortableStreamError("source-unavailable", { cause: make() }); } }); + expect(outcome).toMatchObject({ + status: "refused", step: "S8", reason: "cutover-set-member-evidence-unverifiable", effectBegun: false, + members: [expect.objectContaining({ evidence: "unverifiable", reason, detail: expect.stringContaining("S7 census read") }), expect.objectContaining({ evidence: "unverifiable", reason })], + }); + expectNoEffects(f); + } + }); + + it("cancellation, an unwrapped source failure and a plain error still reject", async () => { + const f = await fixture(); + await expect(f.run({ readDestinationCensus: async () => { throw new PortableStreamError("aborted", { cause: UNHEALTHY() }); } })).rejects.toMatchObject({ code: "aborted" }); + await expect(f.run({ readDestinationCensus: async () => { throw new PortableStreamError("source-unavailable"); } })).rejects.toMatchObject({ code: "source-unavailable" }); + await expect(f.run({ readDestinationCensus: async () => { throw new Error("census exploded"); } })).rejects.toThrow("census exploded"); + await expect(f.run({ captureDestinationIdentity: async () => { throw new Error("capture exploded"); } })).rejects.toThrow("capture exploded"); + }); +}); + +describe("activateCutoverSet: remaining classification seams (#1673)", () => { + it("the real default allocator keeps the set scratch prefix under the LCM home", async () => { + const f = await fixture(); + const dir = createCutoverSetActivateDependencies({ homeDir: f.homeDir }).allocateScratch(f.homeDir); + try { + expect(dirname(dir)).toBe(lcmHomeDir(f.homeDir)); + expect(basename(dir)).toMatch(/^cutover-set-activation-scratch-/u); + } finally { + rmSync(dir, { recursive: true }); + } + }); + + it("an errno from the machine identity read is unresolvable state; an uncoded error rethrows", async () => { + const f = await fixture(); + expect(await f.run({ machineId: () => { throw Object.assign(new Error("permission denied"), { code: "EACCES" }); } })) + .toMatchObject({ status: "refused", step: "S0", reason: "state-unresolvable", effectBegun: false, detail: expect.stringContaining("permission denied") }); + await expect(f.run({ machineId: () => { throw new Error("uncoded"); } })).rejects.toThrow("uncoded"); + }); + + it("a member's identity storage failure does not mask a later member's checkpoint conflict", async () => { + const f = await fixture(); + const [, b] = f.members; + await crashAt(f, "after-checkpoint"); + f.deps.transferRows.set(b!.remoteProjectId, [{ run_id: "migration-copy-" + HASH_A, state: "completed" }]); + let captures = 0; + expect(await f.run({ captureDestinationIdentity: async () => { captures += 1; if (captures === 1) throw UNHEALTHY(); return { version: 1, ...f.deps.identity }; } })) + .toMatchObject({ status: "interrupted", step: "S9", cause: "evidence-conflict", effectBegun: true, detail: expect.stringContaining("copy binding digest differs") }); + }); +}); + +describe("activateCutoverSet: applied is derived from the live config and project map (#1673, #1617)", () => { + const ZERO = "0".repeat(64); + const unreadable = { readArtifact: () => null, readMaintenanceJournal: () => null } as const; + const unresolvableBinding = (onCall: () => void) => ({ + bindings: { recordMigrationSelectionBinding: () => { onCall(); throw new MigrationBindingUnresolvableError("injected"); } }, + }); + + /** A completed own publication whose project-map witness check then interrupts at S16. */ + async function afterPublication(f: Fixture, overrides: Parameters[0] = {}) { + const witness = f.deps.computeProjectMapWitness; + return f.run({ + computeProjectMapWitness: (request) => { + if (request.mode !== "post-publication") return witness(request); + throw new MigrationProjectMapWitnessError("project-map-drift", "injected drift", { drift: "entry-absent" }); + }, + ...overrides, + }); + } + + /** + * A held set at selection-completed (artifact loaded from the store) whose S13a binding write fails; the journal + * readers report `view` from that moment on, which is what the interruption's applied readback sees. + */ + async function viewed(f: Fixture, view: { maintenance?: unknown; publication?: unknown }, extra: Parameters[0] = {}) { + let lie = false; + const maintenance = f.deps.readMaintenanceJournal; + const publication = f.deps.readPublicationJournal; + return f.run({ + bindings: { ...f.deps.bindings, ...unresolvableBinding(() => { lie = true; }).bindings }, + readMaintenanceJournal: (h) => (lie ? (view.maintenance ?? null) as never : maintenance(h)), + readPublicationJournal: (h) => (lie ? (view.publication ?? null) as never : publication(h)), + ...extra, + }); + } + + async function held() { + const f = await fixture(); + await crashAt(f, "after-selection-completed"); + const live = f.deps.readSelectionDigests(f.homeDir); + return { f, live, config: live.configSha256, map: live.projectMapSha256 }; + } + + const interruptedApplied = (applied: string) => ({ status: "interrupted", step: "S13a", applied }); + + it("a completed own publication is yes while the live selection equals the published witnesses and unknown otherwise", async () => { + const f = await fixture(); + expect(await afterPublication(f)).toMatchObject({ status: "interrupted", step: "S16", applied: "yes" }); + const g = await fixture(); + expect(await afterPublication(g, { readSelectionDigests: () => ({ configSha256: ZERO, projectMapSha256: ZERO }) })).toMatchObject({ status: "interrupted", step: "S16", applied: "unknown" }); + const h = await fixture(); + expect(await afterPublication(h, { readSelectionDigests: () => { throw new Error("readback failed"); } })).toMatchObject({ status: "interrupted", step: "S16", applied: "unknown" }); + }); + + it("a yes-phase own journal needs both published witnesses to equal the live selection", async () => { + const { f, config, map } = await held(); + for (const phase of ["config-published", "releasing", "released", "completed"]) { + const own = (extra: object = {}) => ({ publicationId: f.setId, phase, publishedConfigSha256: config, publishedProjectMapSha256: map, ...extra }); + expect(await viewed(f, { publication: own() })).toMatchObject(interruptedApplied("yes")); + expect(await viewed(f, { publication: own({ publishedConfigSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + expect(await viewed(f, { publication: own({ publishedProjectMapSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + expect(await viewed(f, { publication: own({ publishedConfigSha256: null }) })).toMatchObject(interruptedApplied("unknown")); + expect(await viewed(f, { publication: own({ publishedProjectMapSha256: null }) })).toMatchObject(interruptedApplied("unknown")); + } + }); + + it("an in-flight own phase is unknown even when the live selection equals the source", async () => { + const { f, config, map } = await held(); + for (const phase of ["map-publishing", "map-published", "config-publishing"]) { + expect(await viewed(f, { publication: { publicationId: f.setId, phase, expectedConfigSha256: config, expectedProjectMapSha256: map } })).toMatchObject(interruptedApplied("unknown")); + } + }); + + it("a pre-publication or abort phase of the own journal is no only while the live selection equals the expected witnesses", async () => { + const { f, config, map } = await held(); + for (const phase of ["preparing", "prepared", "acquiring", "guarded", "aborting", "config-restoring", "map-restoring", "abort-releasing", "aborted"]) { + const own = (extra: object = {}) => ({ publicationId: f.setId, phase, expectedConfigSha256: config, expectedProjectMapSha256: map, ...extra }); + expect(await viewed(f, { publication: own() })).toMatchObject(interruptedApplied("no")); + expect(await viewed(f, { publication: own({ expectedConfigSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + expect(await viewed(f, { publication: own({ expectedProjectMapSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + } + }); + + it("an own journal whose readback fails is unknown", async () => { + const { f, config, map } = await held(); + const own = { publicationId: f.setId, phase: "completed", publishedConfigSha256: config, publishedProjectMapSha256: map }; + expect(await viewed(f, { publication: own }, { readSelectionDigests: () => { throw new Error("readback failed"); } })).toMatchObject(interruptedApplied("unknown")); + let lie = false; + const publication = f.deps.readPublicationJournal; + expect(await f.run({ + bindings: { ...f.deps.bindings, ...unresolvableBinding(() => { lie = true; }).bindings }, + readPublicationJournal: (h) => { if (lie) throw new Error("journal read failed"); return publication(h); }, + readMaintenanceJournal: (h) => (lie ? null : f.deps.readMaintenanceJournal(h)), + })).toMatchObject(interruptedApplied("unknown")); + }); + + it("a held sqlite window is no only while the live selection equals the artifact's source witness, and a window from another backend is unknown", async () => { + const { f } = await held(); + const records = readCutoverSetActivationRecords({ homeDir: f.homeDir, setId: f.setId }); + if (records.state !== "resolved") throw new Error("records not resolved"); + const source = f.deps.readArtifact({ homeDir: f.homeDir, identityDigest: records.checkpoint!.identityDigest })!.source; + const live = f.deps.readSelectionDigests(f.homeDir); + expect(source.config.presence).toBe("present"); + expect(source.projectMap.presence).toBe("present"); + expect(live.configSha256).toBe(sha256((source.config as { content: Uint8Array }).content)); + expect(live.projectMapSha256).toBe(backendPublicationCanonicalSha256(JSON.parse(Buffer.from((source.projectMap as { content: Uint8Array }).content).toString("utf8")))); + const resolved = { bindings: { ...f.deps.bindings, ...unresolvableBinding(() => undefined).bindings } }; + expect(await f.run(resolved)).toMatchObject(interruptedApplied("no")); + expect(await f.run({ ...resolved, readSelectionDigests: () => ({ ...live, configSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + expect(await f.run({ ...resolved, readSelectionDigests: () => ({ ...live, projectMapSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + expect(await f.run({ ...resolved, readSelectionDigests: () => { throw new Error("readback failed"); } })).toMatchObject(interruptedApplied("unknown")); + expect(await viewed(f, { maintenance: { ...f.journal()!, sourceBackend: "postgresql" } })).toMatchObject(interruptedApplied("unknown")); + }); + + it("an absent or foreign publication journal is no only while the live selection equals the artifact's source witness", async () => { + const { f, live } = await held(); + for (const publication of [null, { publicationId: "other", phase: "completed" }]) { + expect(await viewed(f, { publication })).toMatchObject(interruptedApplied("no")); + expect(await viewed(f, { publication }, { readSelectionDigests: () => ({ ...live, projectMapSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + } + }); + + it("an empty slot is no only while the live selection equals the artifact's source witness", async () => { + const f = await fixture(); + await crashAt(f, "after-selection-completed"); + rmSync(backendPublicationJournalPath(f.homeDir)); + const binding = { bindings: { ...f.deps.bindings, ...unresolvableBinding(() => undefined).bindings } }; + expect(await f.run(binding)).toMatchObject(interruptedApplied("no")); + expect(await f.run({ ...binding, readSelectionDigests: () => ({ configSha256: ZERO, projectMapSha256: ZERO }) })).toMatchObject(interruptedApplied("unknown")); + }); + + it("an absent or foreign publication journal without a source witness is unknown", async () => { + const f = await fixture(); + await crashAt(f, "after-selection-completed"); + expect(await f.run({ ...unreadable, readPublicationJournal: () => null })).toMatchObject({ status: "interrupted", applied: "unknown" }); + expect(await f.run({ ...unreadable, readPublicationJournal: () => ({ publicationId: "other", phase: "completed" }) as never })).toMatchObject({ status: "interrupted", applied: "unknown" }); + }); +}); diff --git a/test/migration/cutover-set-activate-faults.test.ts b/test/migration/cutover-set-activate-faults.test.ts index af16ad819..0bc444242 100644 --- a/test/migration/cutover-set-activate-faults.test.ts +++ b/test/migration/cutover-set-activate-faults.test.ts @@ -102,7 +102,7 @@ describe("activateCutoverSet faults: entry, records and journals", () => { await crashAt(f, "after-checkpoint"); reads = 0; expect(await f.run({ readMaintenanceJournal: (homeDir) => { reads += 1; if (reads === 2) throw new Error("injected under barrier"); return journal(homeDir); } })) - .toMatchObject({ status: "interrupted", step: "S12", cause: "journal-unexpected", applied: "no" }); + .toMatchObject({ status: "interrupted", step: "S12", cause: "journal-unexpected", applied: "unknown" }); }); it("classifies applied from the journals when an interruption is reported", async () => { @@ -113,9 +113,9 @@ describe("activateCutoverSet faults: entry, records and journals", () => { // The journal read inside classifyApplied throws: applied is unknown. expect(await f.run({ readMaintenanceJournal: (homeDir) => { reads += 1; if (reads > 2) throw new Error("later read"); return journal(homeDir); }, readArtifact: () => null })) .toMatchObject({ status: "interrupted", applied: "unknown" }); - // A live publication of another id, and an in-flight own publication. + // A live publication of another id with no source witness in the context, and an in-flight own publication. expect(await f.run({ readArtifact: () => null, readMaintenanceJournal: () => null, readPublicationJournal: () => ({ publicationId: "other", phase: "completed" }) as never })) - .toMatchObject({ status: "interrupted", applied: "no" }); + .toMatchObject({ status: "interrupted", applied: "unknown" }); expect(await f.run({ readArtifact: () => null, readMaintenanceJournal: () => null, readPublicationJournal: () => ({ publicationId: f.setId, phase: "map-published" }) as never })) .toMatchObject({ status: "interrupted", applied: "unknown" }); });