From f68b1988ec8c96dc0ac6b8e13441f4080bf322fb Mon Sep 17 00:00:00 2001 From: andrewn6 <59238070+andrewn6@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:33:41 -0500 Subject: [PATCH 1/6] refactor: split artifact and delta moves into planners the durable checkpoint chain can reuse --- internal/orchestrator/artifacts.go | 52 ++++++++++++++++++------------ internal/orchestrator/delta.go | 31 +++++++++++------- 2 files changed, 51 insertions(+), 32 deletions(-) diff --git a/internal/orchestrator/artifacts.go b/internal/orchestrator/artifacts.go index 9e3819d..e44bd41 100644 --- a/internal/orchestrator/artifacts.go +++ b/internal/orchestrator/artifacts.go @@ -488,24 +488,43 @@ func (fm *FleetManager) planSeedPlacement(ctx context.Context, spec Spec) seedPl // afterwards. the source it pages from is the record itself, so the caller can // pin exactly that snapshot until the target reports its memory resident. func (fm *FleetManager) ensureArtifactOnHost(ctx context.Context, record SnapshotRecord, hostID string, lazy bool) (string, error) { + job, existing, err := fm.planArtifactMove(ctx, record, hostID, lazy) + if err != nil || existing != "" { + return existing, err + } + return fm.runArtifactMove(ctx, job, func(err error) error { + // Cancellation is reported as itself rather than as a transfer failure: + // the caller gave up, the peer did nothing wrong, and a dead-lettered + // "transfer failed" would send an operator looking at the network. + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("copy artifact %s to host %s: %w", record.Digest, hostID, ctxErr) + } + return fmt.Errorf("%w: copy artifact %s from host %s to host %s: %w", + ErrArtifactTransferFailed, record.Digest, job.SourceHostID, hostID, err) + }) +} + +// planArtifactMove works out how record reaches hostID: the id of a copy the +// host already holds (existing), or the move that copies it there. +func (fm *FleetManager) planArtifactMove(ctx context.Context, record SnapshotRecord, hostID string, lazy bool) (job ArtifactMoveJob, existing string, err error) { if hostID == "" { - return "", fmt.Errorf("%w: no target host", ErrArtifactImmovable) + return ArtifactMoveJob{}, "", fmt.Errorf("%w: no target host", ErrArtifactImmovable) } if record.HostID == hostID { - return record.SnapshotID, nil + return ArtifactMoveJob{}, record.SnapshotID, nil } if record.Digest == "" { - return "", fmt.Errorf("%w: snapshot %s has no recorded digest", ErrArtifactImmovable, record.SnapshotID) + return ArtifactMoveJob{}, "", fmt.Errorf("%w: snapshot %s has no recorded digest", ErrArtifactImmovable, record.SnapshotID) } if fm.artifactMover == nil { - return "", fmt.Errorf("%w: no artifact mover is configured", ErrArtifactImmovable) + return ArtifactMoveJob{}, "", fmt.Errorf("%w: no artifact mover is configured", ErrArtifactImmovable) } // a live snapshot moves whole or not at all. its rootfs was copied from a // paused guest with no sync and is only consistent next to the memory // captured with it, so copying the disk half alone would plant something on // the target that looks like a seed and cold-boots into a corrupt guest. if parent := diffParent(record); parent != "" { - return "", fmt.Errorf("%w: snapshot %s is a diff against %s; it only moves onto a copy of its parent (moveDeltaToHost)", + return ArtifactMoveJob{}, "", fmt.Errorf("%w: snapshot %s is a diff against %s; it only moves onto a copy of its parent (moveDeltaToHost)", ErrArtifactImmovable, record.SnapshotID, parent) } // a delta merged onto a version 2 base records the digests its host agent @@ -516,13 +535,13 @@ func (fm *FleetManager) ensureArtifactOnHost(ctx context.Context, record Snapsho if snapshotMetadataString(record.Metadata, "merged_onto") != "" { why = "a delta merged onto a base with no memory chunk table, from a host agent that predates them" } - return "", fmt.Errorf("%w: live snapshot %s has no recorded digests for its memory image (%s)", + return ArtifactMoveJob{}, "", fmt.Errorf("%w: live snapshot %s has no recorded digests for its memory image (%s)", ErrArtifactImmovable, record.SnapshotID, why) } holders, err := fm.HostsHoldingArtifact(ctx, record.TenantID, record.Digest) if err != nil { - return "", err + return ArtifactMoveJob{}, "", err } if record.Kind == SnapshotKindLive { holders = fm.copiesOfLiveSnapshot(ctx, record, holders) @@ -532,15 +551,15 @@ func (fm *FleetManager) ensureArtifactOnHost(ctx context.Context, record Snapsho // transfer; this is the common case on the second create against a host // that pulled the artifact for the first one. fm.touchArtifact(record.SnapshotID, holder.SnapshotID) - return holder.SnapshotID, nil + return ArtifactMoveJob{}, holder.SnapshotID, nil } source, _, err := fm.artifactEndpoints(record, holders, hostID) if err != nil { - return "", err + return ArtifactMoveJob{}, "", err } if lazy && source.snapshotID != record.SnapshotID { - return "", fmt.Errorf("%w: lazy move of %s would page from a copy on host %s rather than the snapshot itself", + return ArtifactMoveJob{}, "", fmt.Errorf("%w: lazy move of %s would page from a copy on host %s rather than the snapshot itself", ErrArtifactImmovable, record.SnapshotID, source.endpoint.HostID) } @@ -555,7 +574,7 @@ func (fm *FleetManager) ensureArtifactOnHost(ctx context.Context, record Snapsho // Both ends are held for the duration: the source is being read and the // destination is being written, and collecting either mid-flight leaves a // torn rootfs that a later create would happily boot. - job := ArtifactMoveJob{ + job = ArtifactMoveJob{ SnapshotID: record.SnapshotID, Digest: record.Digest, SourceHostID: source.endpoint.HostID, @@ -570,16 +589,7 @@ func (fm *FleetManager) ensureArtifactOnHost(ctx context.Context, record Snapsho Kind: record.Kind, SizeBytes: record.SizeBytes, } - return fm.runArtifactMove(ctx, job, func(err error) error { - // Cancellation is reported as itself rather than as a transfer failure: - // the caller gave up, the peer did nothing wrong, and a dead-lettered - // "transfer failed" would send an operator looking at the network. - if ctxErr := ctx.Err(); ctxErr != nil { - return fmt.Errorf("copy artifact %s to host %s: %w", record.Digest, hostID, ctxErr) - } - return fmt.Errorf("%w: copy artifact %s from host %s to host %s: %w", - ErrArtifactTransferFailed, record.Digest, source.endpoint.HostID, hostID, err) - }) + return job, "", nil } // artifactSource is a chosen peer to pull from, plus the id it holds the diff --git a/internal/orchestrator/delta.go b/internal/orchestrator/delta.go index 503c667..bcff7a7 100644 --- a/internal/orchestrator/delta.go +++ b/internal/orchestrator/delta.go @@ -47,20 +47,35 @@ func (fm *FleetManager) forgetSnapshotRecord(ctx context.Context, snapshotID str // none: it can be the base of the next delta or a seed on that host, and // ensureArtifactOnHost refuses it as a source for want of live_files. func (fm *FleetManager) moveDeltaToHost(ctx context.Context, child SnapshotRecord, base, hostID string, dropBase bool) (string, error) { + job, err := fm.planDeltaMove(child, base, hostID, dropBase) + if err != nil { + return "", err + } + return fm.runArtifactMove(ctx, job, func(err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("move delta %s to host %s: %w", child.SnapshotID, hostID, ctxErr) + } + return fmt.Errorf("%w: move delta %s from host %s onto %s on host %s: %w", + ErrArtifactTransferFailed, child.SnapshotID, job.SourceHostID, base, hostID, err) + }) +} + +// planDeltaMove is the move moveDeltaToHost runs. +func (fm *FleetManager) planDeltaMove(child SnapshotRecord, base, hostID string, dropBase bool) (ArtifactMoveJob, error) { if diffParent(child) == "" { - return "", fmt.Errorf("%w: snapshot %s is not a diff", ErrArtifactImmovable, child.SnapshotID) + return ArtifactMoveJob{}, fmt.Errorf("%w: snapshot %s is not a diff", ErrArtifactImmovable, child.SnapshotID) } if fm.artifactMover == nil { - return "", fmt.Errorf("%w: no artifact mover is configured", ErrArtifactImmovable) + return ArtifactMoveJob{}, fmt.Errorf("%w: no artifact mover is configured", ErrArtifactImmovable) } files := liveFiles(child) if len(files) == 0 { - return "", fmt.Errorf("%w: diff snapshot %s has no recorded digests for its delta files", ErrArtifactImmovable, child.SnapshotID) + return ArtifactMoveJob{}, fmt.Errorf("%w: diff snapshot %s has no recorded digests for its delta files", ErrArtifactImmovable, child.SnapshotID) } holders := map[string]ArtifactHolder{child.HostID: {HostID: child.HostID, SnapshotID: child.SnapshotID}} source, _, err := fm.artifactEndpoints(child, holders, hostID) if err != nil { - return "", err + return ArtifactMoveJob{}, err } job := ArtifactMoveJob{ @@ -77,13 +92,7 @@ func (fm *FleetManager) moveDeltaToHost(ctx context.Context, child SnapshotRecor Arch: child.Arch, Kind: SnapshotKindLive, } - return fm.runArtifactMove(ctx, job, func(err error) error { - if ctxErr := ctx.Err(); ctxErr != nil { - return fmt.Errorf("move delta %s to host %s: %w", child.SnapshotID, hostID, ctxErr) - } - return fmt.Errorf("%w: move delta %s from host %s onto %s on host %s: %w", - ErrArtifactTransferFailed, child.SnapshotID, source.endpoint.HostID, base, hostID, err) - }) + return job, nil } // completeLiveFiles reports whether files names a digest for every file a From a87bca77ca5f9e92bd5e9e2e3391f702a43aa543 Mon Sep 17 00:00:00 2001 From: andrewn6 <59238070+andrewn6@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:33:41 -0500 Subject: [PATCH 2/6] feat: run snapshot, restore, the checkpoint chain and snapshot gc on the restate environment object --- internal/durable/artifacts.go | 9 + internal/durable/checkpoint.go | 426 +++++++++++++++++++ internal/durable/destroy.go | 20 +- internal/durable/server.go | 39 ++ internal/durable/snapshot.go | 236 ++++++++++ internal/orchestrator/artifacts_durable.go | 5 + internal/orchestrator/checkpoints.go | 12 +- internal/orchestrator/checkpoints_durable.go | 260 +++++++++++ internal/orchestrator/fleet.go | 5 + internal/orchestrator/fork.go | 2 +- internal/orchestrator/migrate.go | 16 +- internal/orchestrator/snapshot_durable.go | 192 +++++++++ internal/orchestrator/snapshots.go | 414 +++++++++++++----- orchestrator/main.go | 1 + 14 files changed, 1528 insertions(+), 109 deletions(-) create mode 100644 internal/durable/checkpoint.go create mode 100644 internal/durable/snapshot.go create mode 100644 internal/orchestrator/checkpoints_durable.go create mode 100644 internal/orchestrator/snapshot_durable.go diff --git a/internal/durable/artifacts.go b/internal/durable/artifacts.go index 119a116..15b5295 100644 --- a/internal/durable/artifacts.go +++ b/internal/durable/artifacts.go @@ -53,6 +53,15 @@ func moveAbortTimeout(pullTimeout time.Duration) time.Duration { // only a killed invocation leaves them, for an operator to release. func move(ctx restate.ObjectContext, fleet Artifacts, policy RetryPolicy, job orchestrator.ArtifactMoveJob) (string, error) { id, err := moveSteps(ctx, fleet, policy, job) + if job.Notify != "" && (err == nil || restate.IsTerminalError(err)) { + // a checkpoint chain's copy: its vm's object waits for this instead + // of the caller. + res := orchestrator.ChainMoveResult{LocalID: job.LocalID, ID: id} + if err != nil { + res.Err = err.Error() + } + restate.ObjectSend(ctx, EnvironmentObject, job.Notify, "ChainMoved").Send(res) + } if err != nil && restate.IsTerminalError(err) { // the code is what tells the waiting caller that the move itself // ended, whichever restate-server version answers. diff --git a/internal/durable/checkpoint.go b/internal/durable/checkpoint.go new file mode 100644 index 0000000..3a12cb7 --- /dev/null +++ b/internal/durable/checkpoint.go @@ -0,0 +1,426 @@ +package durable + +import ( + "context" + "time" + + restate "github.com/restatedev/sdk-go" + "github.com/restatedev/sdk-go/ingress" + + "github.com/folsomintel/fuse/internal/orchestrator" +) + +// opCheckpoint is the op label on the checkpoint chain's step spans and +// metrics. +const opCheckpoint = "checkpoint" + +// Checkpoints is the part of the fleet the checkpoint chain handlers drive. +// *FleetManager satisfies it. +type Checkpoints interface { + ChainBegin(ctx context.Context, vmID string, chain orchestrator.ChainState) (orchestrator.ChainPlan, error) + ChainMovePlan(ctx context.Context, chain orchestrator.ChainState, plan orchestrator.ChainPlan, child orchestrator.SnapshotRecord) (orchestrator.ChainMove, error) + ChainRetire(ctx context.Context, vmID string, chain orchestrator.ChainState) error + ChainReset(ctx context.Context, vmID, reason string) error + ChainAdvanced(ctx context.Context, vmID string, chain orchestrator.ChainState, diff bool) error + // MirrorChain is not a step: it copies the object's chain into the + // fleet's memory, and runs again on every replay. + MirrorChain(vmID string, chain orchestrator.ChainState, pending bool) +} + +// chainKey is the Environment object state holding the checkpoint chain. +const chainKey = "chain" + +// chainObject is the checkpoint chain as the Environment object keeps it. +type chainObject struct { + Chain orchestrator.ChainState `json:"chain"` + // Pending is a tick's copy to the standby that has not ended yet. + Pending *chainPending `json:"pending,omitempty"` + // HeldUntil is a migrate's hold, unix ms. ticks skip the chain until it + // passes or the migrate releases it. + HeldUntil int64 `json:"held_until,omitempty"` + // Dropped is a vm destroyed while a copy or a hold was in flight: what + // the chain holds is removed when that ends. + Dropped bool `json:"dropped,omitempty"` +} + +// chainPending is a tick's checkpoint on its way to the standby. +type chainPending struct { + Child orchestrator.SnapshotRecord `json:"child"` + Standby string `json:"standby"` + Diff bool `json:"diff,omitempty"` + LocalID string `json:"local_id"` + // Since is when the copy was sent, unix ms. + Since int64 `json:"since"` +} + +// pendingLapse is how long a chain waits on a copy that never reports back +// (an Artifacts invocation an operator killed) before it forgets it. it is +// well past the longest a move's pull can retry. +const pendingLapse = 2 * time.Hour + +// forgetLapsedCopy drops a copy that has not reported back within +// pendingLapse, and the chain with it, deleting nothing: the move may still +// be reading the checkpoint and writing onto the base. it reports whether +// it did. retention gc takes the checkpoint later, artifact gc the base. +func forgetLapsedCopy(ctx restate.ObjectContext, fleet Checkpoints, st *chainObject, now int64) (bool, error) { + if st.Pending == nil || now-st.Pending.Since < pendingLapse.Milliseconds() { + return false, nil + } + vmID := restate.Key(ctx) + if err := chainReset(ctx, fleet, vmID, "checkpoint copy "+st.Pending.LocalID+" never reported back"); err != nil { + return false, err + } + if st.Chain.Base != "" { + restate.ObjectSend(ctx, ArtifactPinObject, st.Chain.Base, "Release").Send(chainHolder(vmID)) + } + st.Pending, st.Chain = nil, orchestrator.ChainState{} + return true, nil +} + +// chainHolder is the ArtifactPin holder of vmID's chain base. +func chainHolder(vmID string) string { + return "chain:" + vmID +} + +func loadChain(ctx restate.ObjectContext) (chainObject, error) { + st, err := restate.Get[*chainObject](ctx, chainKey) + if err != nil || st == nil { + return chainObject{}, err + } + return *st, nil +} + +// saveChain stores st and mirrors it into the fleet. an object holding +// nothing is cleared. +func saveChain(ctx restate.ObjectContext, fleet Checkpoints, st chainObject) { + fleet.MirrorChain(restate.Key(ctx), st.Chain, st.Pending != nil) + if st.Chain.Empty() && st.Pending == nil && st.HeldUntil == 0 && !st.Dropped { + restate.Clear(ctx, chainKey) + return + } + restate.Set(ctx, chainKey, st) +} + +func nowMillis(ctx restate.Context) (int64, error) { + return restate.Run(ctx, func(restate.RunContext) (int64, error) { + return time.Now().UnixMilli(), nil + }, restate.WithName("now")) +} + +// checkpointTick is Environment/{vm_id}.Checkpoint, one step of the vm's +// checkpoint chain: a full checkpoint copied whole to a standby if the chain +// is empty, otherwise a diff against its head merged onto its base there. +// running on the vm's object serializes it with the vm's other lifecycle ops, +// and the chain lives in the object's state, so a restart keeps it. the copy +// is a one-way send to Artifacts, which reports back to ChainMoved, so the +// object is not held for as long as the bytes take to move. +func checkpointTick(ctx restate.ObjectContext, fleet Fleet, policy RetryPolicy) error { + vmID := restate.Key(ctx) + st, err := loadChain(ctx) + if err != nil { + return err + } + fleet.MirrorChain(vmID, st.Chain, st.Pending != nil) + if st.Dropped { + return nil + } + now, err := nowMillis(ctx) + if err != nil { + return err + } + forgot, err := forgetLapsedCopy(ctx, fleet, &st, now) + if err != nil { + return err + } + if forgot { + saveChain(ctx, fleet, st) + return nil + } + if st.Pending != nil { + // the last tick's copy is still running. + return nil + } + if st.HeldUntil > now { + // a migrate of this vm is in progress. + return nil + } + st.HeldUntil = 0 + + plan, err := Run(ctx, opCheckpoint, "plan", func(c context.Context) (orchestrator.ChainPlan, error) { + return fleet.ChainBegin(c, vmID, st.Chain) + }) + if err != nil { + return err + } + if plan.Gone || !plan.Diff { + if err := retireChain(ctx, fleet, vmID, st.Chain); err != nil { + return err + } + st.Chain = orchestrator.ChainState{} + } + if plan.Gone || plan.Standby == "" { + saveChain(ctx, fleet, st) + return nil + } + + child, err := snapshotSteps(ctx, fleet, policy, opCheckpoint, vmID, plan.Options) + if err != nil { + if !restate.IsTerminalError(err) { + return err + } + if plan.Diff { + // the diff was refused (another live snapshot came between) or + // failed: start over with a full one next tick. + if err := chainReset(ctx, fleet, vmID, err.Error()); err != nil { + return err + } + if err := retireChain(ctx, fleet, vmID, st.Chain); err != nil { + return err + } + st.Chain = orchestrator.ChainState{} + } + saveChain(ctx, fleet, st) + return nil + } + + pending := &chainPending{Child: child, Standby: plan.Standby, Diff: plan.Diff} + mv, err := Run(ctx, opCheckpoint, "plan_move", func(c context.Context) (orchestrator.ChainMove, error) { + mv, err := fleet.ChainMovePlan(c, st.Chain, plan, child) + if err != nil { + // planning reads the fleet only; what it refuses, a retry refuses. + return mv, restate.ToTerminalError(err) + } + return mv, nil + }) + if err != nil { + if !restate.IsTerminalError(err) { + return err + } + return chainMoved(ctx, fleet, st, pending, orchestrator.ChainMoveResult{Err: err.Error()}) + } + if mv.Existing != "" { + return chainMoved(ctx, fleet, st, pending, orchestrator.ChainMoveResult{ID: mv.Existing}) + } + job := mv.Job + job.Notify = vmID + pending.LocalID, pending.Since = job.LocalID, now + st.Pending = pending + saveChain(ctx, fleet, st) + restate.ObjectSend(ctx, ArtifactsObject, job.LocalID, "Move").Send(job) + return nil +} + +// chainMovedHandler is Environment/{vm_id}.ChainMoved, the end of a tick's +// copy to the standby. a result for any other copy than the pending one is +// stale and ignored. +func chainMovedHandler(ctx restate.ObjectContext, fleet Fleet, res orchestrator.ChainMoveResult) error { + st, err := loadChain(ctx) + if err != nil { + return err + } + p := st.Pending + if p == nil || p.LocalID != res.LocalID { + fleet.MirrorChain(restate.Key(ctx), st.Chain, p != nil) + return nil + } + st.Pending = nil + return chainMoved(ctx, fleet, st, p, res) +} + +// chainMoved applies the end of a tick's copy: on success the copy becomes +// the chain's base, pinned against gc, and the checkpoint its head; on +// failure the checkpoint is deleted and, for a diff, the chain retired, since +// its base may or may not have been merged and dropped. a copy that ends +// after its vm was destroyed is removed with everything else the chain held. +func chainMoved(ctx restate.ObjectContext, fleet Fleet, st chainObject, p *chainPending, res orchestrator.ChainMoveResult) error { + vmID := restate.Key(ctx) + if st.Dropped { + if res.Err == "" { + if err := retireChain(ctx, fleet, vmID, orchestrator.ChainState{Standby: p.Standby, Base: res.ID}); err != nil { + return err + } + } + if err := retireChain(ctx, fleet, vmID, orchestrator.ChainState{Head: p.Child}); err != nil { + return err + } + return dropChainNow(ctx, fleet, vmID, st) + } + if res.Err != "" { + if err := chainReset(ctx, fleet, vmID, res.Err); err != nil { + return err + } + if err := retireChain(ctx, fleet, vmID, orchestrator.ChainState{Head: p.Child}); err != nil { + return err + } + if p.Diff { + if err := retireChain(ctx, fleet, vmID, st.Chain); err != nil { + return err + } + st.Chain = orchestrator.ChainState{} + } + saveChain(ctx, fleet, st) + return nil + } + + if _, err := restate.Object[restate.Void](ctx, ArtifactPinObject, res.ID, "Pin"). + Request(PinRequest{Holder: chainHolder(vmID)}); err != nil { + return err + } + old := st.Chain + if p.Diff { + // the merge consumed the old base, so only its pin is left to drop. + if old.Base != "" && old.Base != res.ID { + restate.ObjectSend(ctx, ArtifactPinObject, old.Base, "Release").Send(chainHolder(vmID)) + } + st.Chain.Head, st.Chain.Base = p.Child, res.ID + if err := retireChain(ctx, fleet, vmID, orchestrator.ChainState{Head: old.Head}); err != nil { + return err + } + } else { + st.Chain = orchestrator.ChainState{Standby: p.Standby, Head: p.Child, Base: res.ID, Bytes: p.Child.SizeBytes} + } + if err := runStep(ctx, opCheckpoint, "advanced", func(c context.Context) error { + return fleet.ChainAdvanced(c, vmID, st.Chain, p.Diff) + }); err != nil { + return err + } + saveChain(ctx, fleet, st) + return nil +} + +func chainReset(ctx restate.ObjectContext, fleet Checkpoints, vmID, reason string) error { + return runStep(ctx, opCheckpoint, "reset", func(c context.Context) error { + return fleet.ChainReset(c, vmID, reason) + }) +} + +// retireChain removes what chain holds and drops the pin on its base. +func retireChain(ctx restate.ObjectContext, fleet Checkpoints, vmID string, chain orchestrator.ChainState) error { + if chain.Empty() { + return nil + } + if chain.Base != "" { + restate.ObjectSend(ctx, ArtifactPinObject, chain.Base, "Release").Send(chainHolder(vmID)) + } + return runStep(ctx, opCheckpoint, "retire", func(c context.Context) error { + return fleet.ChainRetire(c, vmID, chain) + }) +} + +// dropChainNow retires the whole chain and clears it from the object. +func dropChainNow(ctx restate.ObjectContext, fleet Checkpoints, vmID string, st chainObject) error { + if err := retireChain(ctx, fleet, vmID, st.Chain); err != nil { + return err + } + fleet.MirrorChain(vmID, orchestrator.ChainState{}, false) + restate.Clear(ctx, chainKey) + return nil +} + +// dropChain removes what a gone vm's chain holds: Environment/{vm_id}.RetireChain, +// and the last step of a destroy. a copy or a migrate's hold still in flight +// may be using the base, so the chain is only marked dropped then, and is +// removed when that ends. +func dropChain(ctx restate.ObjectContext, fleet Checkpoints) error { + vmID := restate.Key(ctx) + st, err := loadChain(ctx) + if err != nil { + return err + } + if st.Chain.Empty() && st.Pending == nil && st.HeldUntil == 0 { + return nil + } + now, err := nowMillis(ctx) + if err != nil { + return err + } + if forgot, err := forgetLapsedCopy(ctx, fleet, &st, now); err != nil || forgot { + if err == nil { + fleet.MirrorChain(vmID, orchestrator.ChainState{}, false) + restate.Clear(ctx, chainKey) + } + return err + } + if st.Pending != nil || st.HeldUntil > now { + st.Dropped = true + restate.Set(ctx, chainKey, st) + return nil + } + return dropChainNow(ctx, fleet, vmID, st) +} + +// holdChain is Environment/{vm_id}.HoldChain, taken by a delta migrate. +func holdChain(ctx restate.ObjectContext, fleet Checkpoints, lease time.Duration) (orchestrator.ChainHold, error) { + st, err := loadChain(ctx) + if err != nil { + return orchestrator.ChainHold{}, err + } + if st.Pending != nil || st.Dropped { + return orchestrator.ChainHold{Busy: true}, nil + } + now, err := nowMillis(ctx) + if err != nil { + return orchestrator.ChainHold{}, err + } + st.HeldUntil = now + lease.Milliseconds() + saveChain(ctx, fleet, st) + return orchestrator.ChainHold{Chain: st.Chain}, nil +} + +// releaseChain is Environment/{vm_id}.ReleaseChain. reset forgets the chain +// without deleting its base, which the migrate consumed or will leave to gc. +func releaseChain(ctx restate.ObjectContext, fleet Checkpoints, reset bool) error { + vmID := restate.Key(ctx) + st, err := loadChain(ctx) + if err != nil { + return err + } + st.HeldUntil = 0 + if reset { + if st.Chain.Base != "" { + restate.ObjectSend(ctx, ArtifactPinObject, st.Chain.Base, "Release").Send(chainHolder(vmID)) + } + st.Chain = orchestrator.ChainState{} + } + if st.Dropped && st.Pending == nil { + return dropChainNow(ctx, fleet, vmID, st) + } + saveChain(ctx, fleet, st) + return nil +} + +// HoldRequest is the input of HoldChain. +type HoldRequest struct { + Lease time.Duration `json:"lease"` +} + +// Checkpoint runs Environment/{vmID}.Checkpoint and waits for it. +func (c *Client) Checkpoint(ctx context.Context, vmID string) error { + _, err := ingress.Object[restate.Void, restate.Void](c.longIngress, EnvironmentObject, vmID, "Checkpoint"). + Request(ctx, restate.Void{}) + return err +} + +// HoldChain runs Environment/{vmID}.HoldChain. it waits behind a tick that is +// taking its checkpoint, so it goes through the client with no request +// timeout; ctx bounds the wait. +func (c *Client) HoldChain(ctx context.Context, vmID string, lease time.Duration) (orchestrator.ChainHold, error) { + return ingress.Object[HoldRequest, orchestrator.ChainHold](c.longIngress, EnvironmentObject, vmID, "HoldChain"). + Request(ctx, HoldRequest{Lease: lease}) +} + +// ReleaseChain enqueues Environment/{vmID}.ReleaseChain without waiting: it +// may queue behind a long op on the vm, and a hold that is not released only +// lapses on its lease. +func (c *Client) ReleaseChain(ctx context.Context, vmID string, reset bool) error { + _, err := ingress.ObjectSend[bool](c.ingress, EnvironmentObject, vmID, "ReleaseChain"). + Send(ctx, reset) + return err +} + +// SendRetireChain enqueues Environment/{vmID}.RetireChain. +func (c *Client) SendRetireChain(ctx context.Context, vmID string) error { + _, err := ingress.ObjectSend[restate.Void](c.ingress, EnvironmentObject, vmID, "RetireChain"). + Send(ctx, restate.Void{}) + return err +} diff --git a/internal/durable/destroy.go b/internal/durable/destroy.go index f9e190e..bf4acaf 100644 --- a/internal/durable/destroy.go +++ b/internal/durable/destroy.go @@ -27,7 +27,8 @@ type Destroyer interface { // crash the handler resumes at the first unfinished one. the host destroy // retries under policy; when it runs out the vm is left destroy_failed with a // dead letter, and the invocation fails with the last error. -func destroy(ctx restate.ObjectContext, fleet Destroyer, policy RetryPolicy, req orchestrator.DestroyRequest) error { +// the last step drops what the vm's checkpoint chain held. +func destroy(ctx restate.ObjectContext, fleet destroyFleet, policy RetryPolicy, req orchestrator.DestroyRequest) error { vmID := restate.Key(ctx) target, err := Run(ctx, opDestroy, "begin", func(c context.Context) (orchestrator.DestroyTarget, error) { @@ -44,7 +45,8 @@ func destroy(ctx restate.ObjectContext, fleet Destroyer, policy RetryPolicy, req if !target.Tracked { if req.HostID == "" { // neither tracked nor placed: an earlier destroy finished it. - return nil + // any checkpoint chain the object still keeps for it goes too. + return dropChain(ctx, fleet) } hostID = req.HostID } @@ -78,9 +80,19 @@ func destroy(ctx restate.ObjectContext, fleet Destroyer, policy RetryPolicy, req }); err != nil { return err } - return step(ctx, "finish", func(c context.Context) error { + if err := step(ctx, "finish", func(c context.Context) error { return fleet.DestroyFinish(c, vmID, req.Reason) - }) + }); err != nil { + return err + } + return dropChain(ctx, fleet) +} + +// destroyFleet is what the destroy handler drives: its own steps, and the +// checkpoint chain it drops at the end. +type destroyFleet interface { + Destroyer + Checkpoints } // step is Run for a step with no result. diff --git a/internal/durable/server.go b/internal/durable/server.go index 2321796..d2b9b8d 100644 --- a/internal/durable/server.go +++ b/internal/durable/server.go @@ -42,8 +42,15 @@ type Fleet interface { GetHost(hostID string) (orchestrator.Host, bool) Destroyer Artifacts + Snapshots + Checkpoints } +// snapshotAbortTimeout is how long restate lets a snapshot, restore or +// checkpoint attempt run without journal progress before it aborts and +// retries it. a live snapshot writes the whole memory image in one step. +const snapshotAbortTimeout = 30 * time.Minute + // PingResult is what Ping returns. it never carries more than whether the // fleet knows the key: a Host carries its agent token, which must not leave // through ingress. @@ -66,6 +73,38 @@ func NewEndpoint(cfg Config, fleet Fleet, logger *slog.Logger) *server.Restate { Handler("Destroy", restate.NewObjectHandler( func(ctx restate.ObjectContext, req orchestrator.DestroyRequest) (restate.Void, error) { return restate.Void{}, destroy(ctx, fleet, servicePolicy(cfg, EnvironmentObject), req) + })). + Handler("Snapshot", restate.NewObjectHandler( + func(ctx restate.ObjectContext, opts orchestrator.SnapshotOptions) (orchestrator.SnapshotRecord, error) { + return snapshot(ctx, fleet, servicePolicy(cfg, EnvironmentObject), opts) + }, restate.WithAbortTimeout(snapshotAbortTimeout))). + Handler("Restore", restate.NewObjectHandler( + func(ctx restate.ObjectContext, snapshotID string) (restate.Void, error) { + return restate.Void{}, restore(ctx, fleet, servicePolicy(cfg, EnvironmentObject), snapshotID) + }, restate.WithAbortTimeout(snapshotAbortTimeout))). + Handler("CollectSnapshot", restate.NewObjectHandler( + func(ctx restate.ObjectContext, snapshotID string) (restate.Void, error) { + return restate.Void{}, collectSnapshot(ctx, fleet, servicePolicy(cfg, EnvironmentObject), snapshotID) + })). + Handler("Checkpoint", restate.NewObjectHandler( + func(ctx restate.ObjectContext, _ restate.Void) (restate.Void, error) { + return restate.Void{}, checkpointTick(ctx, fleet, servicePolicy(cfg, EnvironmentObject)) + }, restate.WithAbortTimeout(snapshotAbortTimeout))). + Handler("ChainMoved", restate.NewObjectHandler( + func(ctx restate.ObjectContext, res orchestrator.ChainMoveResult) (restate.Void, error) { + return restate.Void{}, chainMovedHandler(ctx, fleet, res) + })). + Handler("HoldChain", restate.NewObjectHandler( + func(ctx restate.ObjectContext, req HoldRequest) (orchestrator.ChainHold, error) { + return holdChain(ctx, fleet, req.Lease) + })). + Handler("ReleaseChain", restate.NewObjectHandler( + func(ctx restate.ObjectContext, reset bool) (restate.Void, error) { + return restate.Void{}, releaseChain(ctx, fleet, reset) + })). + Handler("RetireChain", restate.NewObjectHandler( + func(ctx restate.ObjectContext, _ restate.Void) (restate.Void, error) { + return restate.Void{}, dropChain(ctx, fleet) })) host := restate.NewObject(HostObject, retryOption(cfg, HostObject)). diff --git a/internal/durable/snapshot.go b/internal/durable/snapshot.go new file mode 100644 index 0000000..798f123 --- /dev/null +++ b/internal/durable/snapshot.go @@ -0,0 +1,236 @@ +package durable + +import ( + "context" + "errors" + "fmt" + + restate "github.com/restatedev/sdk-go" + "github.com/restatedev/sdk-go/ingress" + + "github.com/folsomintel/fuse/internal/orchestrator" +) + +// op labels on the step spans and metrics. +const ( + opSnapshot = "snapshot" + opRestore = "restore" + opSnapshotG = "snapshot_gc" +) + +// Snapshots is the part of the fleet the snapshot, restore and snapshot gc +// handlers drive, one method per step. *FleetManager satisfies it. +type Snapshots interface { + SnapshotBegin(ctx context.Context, vmID string, opts orchestrator.SnapshotOptions) (orchestrator.SnapshotPlan, error) + SnapshotCheckpoint(ctx context.Context, vmID string, plan orchestrator.SnapshotPlan) (orchestrator.Checkpoint, error) + SnapshotRecordCreating(ctx context.Context, vmID string, plan orchestrator.SnapshotPlan, created orchestrator.Checkpoint) (orchestrator.SnapshotRecord, error) + SnapshotList(ctx context.Context, vmID, snapshotID string) (orchestrator.Checkpoint, error) + SnapshotReady(ctx context.Context, record orchestrator.SnapshotRecord, listed orchestrator.Checkpoint) (orchestrator.SnapshotRecord, error) + SnapshotAbort(ctx context.Context, vmID string, plan orchestrator.SnapshotPlan, created orchestrator.Checkpoint, cause string) error + + RestoreValidate(ctx context.Context, vmID, snapshotID string) (orchestrator.SnapshotRecord, error) + RestoreBegin(ctx context.Context, record orchestrator.SnapshotRecord) error + RestoreOnHost(ctx context.Context, vmID, snapshotID string) error + RestoreFinish(ctx context.Context, record orchestrator.SnapshotRecord) error + RestoreFailed(ctx context.Context, record orchestrator.SnapshotRecord, lastErr string) error + + SnapshotCollect(ctx context.Context, snapshotID string) error + SnapshotCollectFailed(ctx context.Context, snapshotID, lastErr string) error +} + +// fleetErrorBase is the first error code a fleet sentinel error crosses +// restate as: orchestrator.SnapshotFleetErrors[i] is fleetErrorBase+i. the +// api maps the sentinel, so the code only has to bring it back. codes below +// 480 are restate's own (404 not found, 470 not ready). +const fleetErrorBase restate.Code = 480 + +// permanent marks err terminal when a retry cannot change it, carrying a +// fleet sentinel as its code. the code is journaled with the error, so a +// replay that returns the recorded failure still carries it. +func permanent(err error) error { + if err == nil || !orchestrator.PermanentSnapshotError(err) { + return err + } + for i, s := range orchestrator.SnapshotFleetErrors { + if errors.Is(err, s) { + return restate.ToTerminalError(err, restate.WithErrorCode(fleetErrorBase+restate.Code(i))) + } + } + return restate.ToTerminalError(err) +} + +// fleetError turns a handler's failure back into the fleet sentinel its code +// names, so the api answers it with the same status as in mode off. +func fleetError(err error) error { + var ierr ingress.Error + if !errors.As(err, &ierr) { + return err + } + i := int(ierr.Code()) - int(fleetErrorBase) + if i < 0 || i >= len(orchestrator.SnapshotFleetErrors) { + return err + } + return fmt.Errorf("%w: %s", orchestrator.SnapshotFleetErrors[i], ierr.Message()) +} + +// retryUnder is policy as the options of one step. +func retryUnder(policy RetryPolicy) []restate.RunOption { + return []restate.RunOption{ + restate.WithInitialRetryInterval(policy.InitialInterval), + restate.WithMaxRetryInterval(policy.MaxInterval), + restate.WithMaxRetryAttempts(uint(policy.MaxAttempts)), + } +} + +// snapshot is Environment/{vm_id}.Snapshot. +func snapshot(ctx restate.ObjectContext, fleet Snapshots, policy RetryPolicy, opts orchestrator.SnapshotOptions) (orchestrator.SnapshotRecord, error) { + return snapshotSteps(ctx, fleet, policy, opSnapshot, restate.Key(ctx), opts) +} + +// snapshotSteps takes one snapshot of vmID, each step journaled: begin +// (checks and quota), checkpoint on the host, the row in creating, the +// host's listing, ready. the checkpoint is keyed, so a retry or a replay +// after a crash gets the first checkpoint back and the row is written for +// it. a record that cannot be finished deletes the host artifact and marks +// the row error, so nothing is left on the host that no row tracks. the +// checkpoint chain takes its snapshots through the same steps. +func snapshotSteps(ctx restate.ObjectContext, fleet Snapshots, policy RetryPolicy, op, vmID string, opts orchestrator.SnapshotOptions) (orchestrator.SnapshotRecord, error) { + retry := retryUnder(policy) + plan, err := Run(ctx, op, "begin", func(c context.Context) (orchestrator.SnapshotPlan, error) { + plan, err := fleet.SnapshotBegin(c, vmID, opts) + return plan, permanent(err) + }) + if err != nil { + return orchestrator.SnapshotRecord{}, err + } + created, err := Run(ctx, op, "checkpoint", func(c context.Context) (orchestrator.Checkpoint, error) { + created, err := fleet.SnapshotCheckpoint(c, vmID, plan) + return created, permanent(err) + }, retry...) + if err != nil { + return orchestrator.SnapshotRecord{}, err + } + + record, err := Run(ctx, op, "record", func(c context.Context) (orchestrator.SnapshotRecord, error) { + return fleet.SnapshotRecordCreating(c, vmID, plan, created) + }, retry...) + var listed orchestrator.Checkpoint + if err == nil { + listed, err = Run(ctx, op, "list", func(c context.Context) (orchestrator.Checkpoint, error) { + return fleet.SnapshotList(c, vmID, created.ID) + }, retry...) + } + if err == nil { + record, err = Run(ctx, op, "ready", func(c context.Context) (orchestrator.SnapshotRecord, error) { + return fleet.SnapshotReady(c, record, listed) + }, retry...) + } + if err != nil { + if !restate.IsTerminalError(err) { + // suspension: restate resumes the snapshot here. + return orchestrator.SnapshotRecord{}, err + } + cause := err.Error() + if _, aerr := Run(ctx, op, "abort", func(c context.Context) (restate.Void, error) { + return restate.Void{}, fleet.SnapshotAbort(c, vmID, plan, created, cause) + }, retry...); aerr != nil && !restate.IsTerminalError(aerr) { + return orchestrator.SnapshotRecord{}, aerr + } + return orchestrator.SnapshotRecord{}, err + } + return record, nil +} + +// restore is Environment/{vm_id}.Restore: validate (the row and the host's +// copy), restoring, the restore on the host, ready. a restart replays to the +// first unfinished step, so the restore completes instead of leaving the row +// in restoring. a host restore that fails for good marks the row error. +func restore(ctx restate.ObjectContext, fleet Snapshots, policy RetryPolicy, snapshotID string) error { + vmID := restate.Key(ctx) + retry := retryUnder(policy) + record, err := Run(ctx, opRestore, "validate", func(c context.Context) (orchestrator.SnapshotRecord, error) { + record, err := fleet.RestoreValidate(c, vmID, snapshotID) + return record, permanent(err) + }) + if err != nil { + return err + } + if err := runStep(ctx, opRestore, "restoring", func(c context.Context) error { + return fleet.RestoreBegin(c, record) + }, retry...); err != nil { + return err + } + hostErr := runStep(ctx, opRestore, "restore", func(c context.Context) error { + return permanent(fleet.RestoreOnHost(c, vmID, snapshotID)) + }, retry...) + if hostErr != nil { + if !restate.IsTerminalError(hostErr) { + return hostErr + } + if err := runStep(ctx, opRestore, "failed", func(c context.Context) error { + return fleet.RestoreFailed(c, record, hostErr.Error()) + }, retry...); err != nil && !restate.IsTerminalError(err) { + return err + } + return hostErr + } + return runStep(ctx, opRestore, "finish", func(c context.Context) error { + return fleet.RestoreFinish(c, record) + }, retry...) +} + +// collectSnapshot is Environment/{vm_id}.CollectSnapshot, the deletion of +// one snapshot retention gc marked deleting. the deletion retries under the +// Environment policy; when that runs out the row moves to error, and a later +// sweep marks it again. +func collectSnapshot(ctx restate.ObjectContext, fleet Snapshots, policy RetryPolicy, snapshotID string) error { + retry := retryUnder(policy) + err := runStep(ctx, opSnapshotG, "collect", func(c context.Context) error { + return fleet.SnapshotCollect(c, snapshotID) + }, retry...) + if err == nil || !restate.IsTerminalError(err) { + return err + } + lastErr := err.Error() + if ferr := runStep(ctx, opSnapshotG, "collect_failed", func(c context.Context) error { + return fleet.SnapshotCollectFailed(c, snapshotID, lastErr) + }, retry...); ferr != nil && !restate.IsTerminalError(ferr) { + return ferr + } + // the failure is recorded on the row; the invocation itself is done. + return nil +} + +// runStep is Run for a step with no result, under op. +func runStep(ctx restate.Context, op, name string, fn func(context.Context) error, opts ...restate.RunOption) error { + _, err := Run(ctx, op, name, func(c context.Context) (restate.Void, error) { + return restate.Void{}, fn(c) + }, opts...) + return err +} + +// Snapshot runs Environment/{vmID}.Snapshot and waits for it. a live +// snapshot writes the whole memory image, so it goes through the client with +// no request timeout of its own; ctx bounds the wait. +func (c *Client) Snapshot(ctx context.Context, vmID string, opts orchestrator.SnapshotOptions) (orchestrator.SnapshotRecord, error) { + rec, err := ingress.Object[orchestrator.SnapshotOptions, orchestrator.SnapshotRecord](c.longIngress, EnvironmentObject, vmID, "Snapshot"). + Request(ctx, opts) + return rec, fleetError(err) +} + +// Restore runs Environment/{vmID}.Restore and waits for it. +func (c *Client) Restore(ctx context.Context, vmID, snapshotID string) error { + _, err := ingress.Object[string, restate.Void](c.longIngress, EnvironmentObject, vmID, "Restore"). + Request(ctx, snapshotID) + return fleetError(err) +} + +// SendCollectSnapshot enqueues Environment/{vmID}.CollectSnapshot. restate +// runs one invocation per key. +func (c *Client) SendCollectSnapshot(ctx context.Context, vmID, snapshotID, key string) error { + _, err := ingress.ObjectSend[string](c.ingress, EnvironmentObject, vmID, "CollectSnapshot"). + Send(ctx, snapshotID, restate.WithIdempotencyKey(key)) + return err +} + +var _ orchestrator.DurableSnapshots = (*Client)(nil) diff --git a/internal/orchestrator/artifacts_durable.go b/internal/orchestrator/artifacts_durable.go index 6fdb6d5..640d8ea 100644 --- a/internal/orchestrator/artifacts_durable.go +++ b/internal/orchestrator/artifacts_durable.go @@ -30,6 +30,11 @@ type ArtifactMoveJob struct { Base string `json:"base,omitempty"` DropBase bool `json:"drop_base,omitempty"` + // Notify names the vm whose checkpoint chain waits on this move. its end + // is sent to Environment/{Notify}.ChainMoved instead of being awaited, + // so the chain does not hold the vm's object for the whole copy. + Notify string `json:"notify,omitempty"` + // what the copy's record takes from the moved record. TenantID string `json:"tenant_id"` LayerKey string `json:"layer_key,omitempty"` diff --git a/internal/orchestrator/checkpoints.go b/internal/orchestrator/checkpoints.go index 433b123..17c3292 100644 --- a/internal/orchestrator/checkpoints.go +++ b/internal/orchestrator/checkpoints.go @@ -31,6 +31,10 @@ type checkpointChain struct { // size, so it is not updated per tick (the merged record only carries // the delta that moved). bytes int64 + // pending is set on restate while a tick's copy to the standby runs. + // there the chain's state lives on the Environment object, and this + // struct only mirrors it (MirrorChain). + pending bool } // setStandbyLocked records where the chain's base lives and how big it is. @@ -161,6 +165,10 @@ func (fm *FleetManager) checkpointLoop(ctx context.Context) { case <-ctx.Done(): return case <-tick.C: + if d := fm.durableSnapshotsClient(); d != nil { + fm.checkpointTickDurable(ctx, d) + continue + } fm.checkpointTick(ctx) } } @@ -349,7 +357,7 @@ func (fm *FleetManager) advanceChain(ctx context.Context, vmID string) { } } if keep { - child, err := fm.CreateSnapshot(ctx, vmID, SnapshotOptions{ + child, err := fm.snapshotInProcess(ctx, vmID, SnapshotOptions{ Comment: "checkpoint", Mode: SnapshotModeAuto, RetentionUntil: &retention, Live: true, Diff: true, Parent: c.head.SnapshotID, NoLineage: true, }) @@ -394,7 +402,7 @@ func (fm *FleetManager) advanceChain(ctx context.Context, vmID string) { if standby == "" { return } - full, err := fm.CreateSnapshot(ctx, vmID, SnapshotOptions{ + full, err := fm.snapshotInProcess(ctx, vmID, SnapshotOptions{ Comment: "checkpoint", Mode: SnapshotModeAuto, RetentionUntil: &retention, Live: true, NoLineage: true, }) diff --git a/internal/orchestrator/checkpoints_durable.go b/internal/orchestrator/checkpoints_durable.go new file mode 100644 index 0000000..a568b84 --- /dev/null +++ b/internal/orchestrator/checkpoints_durable.go @@ -0,0 +1,260 @@ +package orchestrator + +import ( + "context" + "time" +) + +// chainHoldLease bounds how long a migrate keeps the checkpoint loop off its +// vm's chain on restate. a migrate that dies with the process leaves the hold +// to lapse instead of stopping the chain for good. +const chainHoldLease = time.Hour + +// checkpointCallWait bounds how long a tick waits on one vm's checkpoint +// handler. the handler keeps running if the wait ends. +const checkpointCallWait = 10 * time.Minute + +// ChainState is one vm's checkpoint chain as the restate Environment object +// keeps it, so a restart loses neither the chain nor what it holds on the +// standby. see checkpointChain for what each field is. +type ChainState struct { + Standby string `json:"standby,omitempty"` + Head SnapshotRecord `json:"head"` + Base string `json:"base,omitempty"` + Bytes int64 `json:"bytes,omitempty"` +} + +// Empty reports whether the chain holds nothing. +func (c ChainState) Empty() bool { + return c.Head.SnapshotID == "" && c.Base == "" +} + +// ChainPlan is what one checkpoint tick does, as ChainBegin settled it. +type ChainPlan struct { + // Gone is a vm the fleet no longer tracks: its chain is retired. + Gone bool `json:"gone,omitempty"` + // Diff extends the chain: a diff against its head, merged onto its base + // on Standby. otherwise any chain is retired, and a full checkpoint is + // copied to Standby, or nothing is taken when Standby is "". + Diff bool `json:"diff,omitempty"` + Standby string `json:"standby,omitempty"` + Options SnapshotOptions `json:"options"` +} + +// ChainMove is how a tick's checkpoint reaches the standby: a copy the +// standby already holds (Existing), or Job. +type ChainMove struct { + Job ArtifactMoveJob `json:"job"` + Existing string `json:"existing,omitempty"` +} + +// ChainHold is what HoldChain hands a migrate. +type ChainHold struct { + Busy bool `json:"busy,omitempty"` + Chain ChainState `json:"chain"` +} + +// ChainMoveResult is the end of a chain's move, sent by the Artifacts move +// to Environment/{vm_id}.ChainMoved: the committed id, or the move's error. +type ChainMoveResult struct { + LocalID string `json:"local_id"` + ID string `json:"id,omitempty"` + Err string `json:"err,omitempty"` +} + +// checkpointTickDurable is checkpointTick on restate: each candidate's tick +// runs as Environment/{vm_id}.Checkpoint, serialized with the vm's other +// lifecycle ops, and its chain lives in that object's state. vms still go one +// at a time: the next is not started until this one's copy to its standby +// has ended. +func (fm *FleetManager) checkpointTickDurable(ctx context.Context, d DurableSnapshots) { + for _, vmID := range fm.checkpointCandidates() { + if ctx.Err() != nil { + return + } + cctx, cancel := context.WithTimeout(ctx, checkpointCallWait) + err := d.Checkpoint(cctx, vmID) + cancel() + if err != nil { + fm.logger.Warn("durable checkpoint tick failed", "vm", vmID, "err", err) + continue + } + fm.waitChainSettled(ctx, vmID) + } + + // what the chain of a vm that is gone still holds is removed by its + // object; a destroy on restate already did it, this catches the rest. + fm.mu.RLock() + fm.chainsMu.Lock() + var gone []string + for vmID := range fm.chains { + if _, ok := fm.vms[vmID]; !ok { + gone = append(gone, vmID) + } + } + fm.chainsMu.Unlock() + fm.mu.RUnlock() + for _, vmID := range gone { + sctx, cancel := context.WithTimeout(ctx, 10*time.Second) + err := d.SendRetireChain(sctx, vmID) + cancel() + if err != nil { + fm.logger.Warn("send checkpoint chain retire failed, the next tick retries", "vm", vmID, "err", err) + continue + } + fm.MirrorChain(vmID, ChainState{}, false) + } +} + +// waitChainSettled waits until vmID's chain has no copy in flight, ctx ends, +// or durableMoveWait passes. +func (fm *FleetManager) waitChainSettled(ctx context.Context, vmID string) { + deadline := time.NewTimer(durableMoveWait) + defer deadline.Stop() + poll := time.NewTicker(500 * time.Millisecond) + defer poll.Stop() + for fm.chainPending(vmID) { + select { + case <-ctx.Done(): + return + case <-deadline.C: + return + case <-poll.C: + } + } +} + +func (fm *FleetManager) chainPending(vmID string) bool { + fm.chainsMu.Lock() + defer fm.chainsMu.Unlock() + c, ok := fm.chains[vmID] + return ok && c.pending +} + +// MirrorChain copies a chain's durable state into fm.chains, which the +// standby picker and migrate target choice read. it is not journaled: the +// handlers call it on every replay, so it always ends at the object's state. +func (fm *FleetManager) MirrorChain(vmID string, chain ChainState, pending bool) { + fm.chainsMu.Lock() + defer fm.chainsMu.Unlock() + if chain.Empty() && chain.Standby == "" && !pending { + delete(fm.chains, vmID) + return + } + c, ok := fm.chains[vmID] + if !ok { + c = &checkpointChain{} + fm.chains[vmID] = c + } + c.standby, c.bytes, c.head, c.base, c.pending = chain.Standby, chain.Bytes, chain.Head, chain.Base, pending +} + +// the methods below are the steps of the durable checkpoint chain. each runs +// inside a journaled restate step and must be safe to run again. + +// ChainBegin settles what this tick does with vmID's chain, by the rules +// advanceChain follows in process. +func (fm *FleetManager) ChainBegin(ctx context.Context, vmID string, chain ChainState) (ChainPlan, error) { + fm.mu.RLock() + v, ok := fm.vms[vmID] + srcHostID := "" + if ok { + srcHostID = v.hostID + } + fm.mu.RUnlock() + if !ok { + return ChainPlan{Gone: true}, nil + } + retention := time.Now().Add(4 * fm.checkpointInterval) + + keep := chain.Base != "" && chain.Standby != "" && chain.Standby != srcHostID + if keep { + if reason := fm.standbyRejectionFor(vmID, chain.Standby); reason != "" { + fm.logger.InfoContext(ctx, "checkpoint standby no longer fits; re-picking", "vm", vmID, "host", chain.Standby, "reason", reason) + fm.appendEventBackground("vm", vmID, "vm.checkpoint_chain_reset", map[string]any{ + "error": "standby host " + chain.Standby + " " + reason, "standby_host_id": chain.Standby, + }) + keep = false + } + } + if keep { + return ChainPlan{Diff: true, Standby: chain.Standby, Options: SnapshotOptions{ + Comment: "checkpoint", Mode: SnapshotModeAuto, RetentionUntil: &retention, + Live: true, Diff: true, Parent: chain.Head.SnapshotID, NoLineage: true, + }}, nil + } + return ChainPlan{Standby: fm.pickStandby(vmID, srcHostID), Options: SnapshotOptions{ + Comment: "checkpoint", Mode: SnapshotModeAuto, RetentionUntil: &retention, + Live: true, NoLineage: true, + }}, nil +} + +// ChainMovePlan works out how the tick's checkpoint reaches the standby: a +// delta merged onto the chain's base, or a full copy. +func (fm *FleetManager) ChainMovePlan(ctx context.Context, chain ChainState, plan ChainPlan, child SnapshotRecord) (ChainMove, error) { + if plan.Diff { + job, err := fm.planDeltaMove(child, chain.Base, plan.Standby, true) + return ChainMove{Job: job}, err + } + job, existing, err := fm.planArtifactMove(ctx, child, plan.Standby, false) + return ChainMove{Job: job, Existing: existing}, err +} + +// ChainRetire removes what a chain holds: its base on the standby, with its +// record, and its head on the source. failures are logged, never returned: +// what is left is retention's or artifact gc's. +func (fm *FleetManager) ChainRetire(ctx context.Context, vmID string, chain ChainState) error { + if chain.Base != "" { + if err := fm.deleteFreeArtifact(ctx, chain.Standby, chain.Base); err != nil && !IsNotFound(err) { + fm.logger.WarnContext(ctx, "delete stale checkpoint base failed", "vm", vmID, "host", chain.Standby, "snapshot", chain.Base, "err", err) + } + fm.forgetSnapshotRecord(ctx, chain.Base) + } + fm.deleteCheckpoint(ctx, chain.Head) + return nil +} + +// ChainReset records why a chain was retired. +func (fm *FleetManager) ChainReset(ctx context.Context, vmID, reason string) error { + fm.logger.InfoContext(ctx, "checkpoint chain reset", "vm", vmID, "reason", reason) + fm.appendEvent(ctx, "vm", vmID, "vm.checkpoint_chain_reset", map[string]any{"error": reason}) + return nil +} + +// ChainAdvanced records a chain that took a step: its head is the new +// checkpoint and its base the copy on the standby. +func (fm *FleetManager) ChainAdvanced(ctx context.Context, vmID string, chain ChainState, diff bool) error { + fm.touchArtifact(chain.Head.SnapshotID, chain.Base) + fm.appendEvent(ctx, "vm", vmID, "vm.checkpointed", map[string]any{ + "snapshot_id": chain.Head.SnapshotID, "standby_host_id": chain.Standby, "base": chain.Base, "diff": diff, + }) + return nil +} + +// holdDurableChain holds vmID's chain on restate for a delta migrate and +// returns a copy of it, or nil when it cannot be held: restate refused, or a +// tick's copy is still running. a migrate without the hold takes a full seed +// instead, which only costs the chain its next diff. +func (fm *FleetManager) holdDurableChain(ctx context.Context, d DurableSnapshots, vmID string) *checkpointChain { + hold, err := d.HoldChain(ctx, vmID, chainHoldLease) + if err != nil { + fm.logger.WarnContext(ctx, "hold checkpoint chain failed; migrating without it", "vm", vmID, "err", err) + return nil + } + if hold.Busy { + fm.logger.InfoContext(ctx, "checkpoint chain copy in flight; migrating without it", "vm", vmID) + return nil + } + c := hold.Chain + return &checkpointChain{standby: c.Standby, head: c.Head, base: c.Base, bytes: c.Bytes} +} + +// releaseDurableChain ends a migrate's hold. reset forgets the chain, whose +// base the migrate consumed or put out of step. +func (fm *FleetManager) releaseDurableChain(d DurableSnapshots, vmID string, reset bool) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := d.ReleaseChain(ctx, vmID, reset); err != nil { + fm.logger.Warn("release checkpoint chain failed; the hold lapses on its lease", "vm", vmID, "err", err) + } +} diff --git a/internal/orchestrator/fleet.go b/internal/orchestrator/fleet.go index 683a955..c0128da 100644 --- a/internal/orchestrator/fleet.go +++ b/internal/orchestrator/fleet.go @@ -625,6 +625,11 @@ type FleetManager struct { // nil in mode off. see artifacts_durable.go. durableArtifacts DurableArtifacts + // durableSnapshots runs snapshots, restores, the checkpoint chain and + // retention gc on restate; nil in mode off. see snapshot_durable.go and + // checkpoints_durable.go. + durableSnapshots DurableSnapshots + cancel context.CancelFunc done chan struct{} } diff --git a/internal/orchestrator/fork.go b/internal/orchestrator/fork.go index 6a716df..744629d 100644 --- a/internal/orchestrator/fork.go +++ b/internal/orchestrator/fork.go @@ -187,7 +187,7 @@ func (fm *FleetManager) forkEnvironment(ctx context.Context, srcVMID string, opt // belongs to the source and is ready. var seed SnapshotRecord if opts.ReuseSnapshotID == "" { - created, err := fm.CreateSnapshot(ctx, srcVMID, SnapshotOptions{Comment: opts.Comment}) + created, err := fm.snapshotInProcess(ctx, srcVMID, SnapshotOptions{Comment: opts.Comment}) if err != nil { return "", err } diff --git a/internal/orchestrator/migrate.go b/internal/orchestrator/migrate.go index 1eabe1d..b044869 100644 --- a/internal/orchestrator/migrate.go +++ b/internal/orchestrator/migrate.go @@ -192,7 +192,17 @@ func (fm *FleetManager) migrateVM(ctx context.Context, vmID string, opts Migrate srcDelta, _ := srcEnv.(DeltaSnapshotCapable) delta := opts.Live && !opts.Lazy && srcDelta != nil && fm.artifactMover != nil var chain *checkpointChain - if delta { + if d := fm.durableSnapshotsClient(); delta && d != nil { + // on restate the chain lives on the vm's Environment object, and + // the hold is what keeps the checkpoint loop off it. + chain = fm.holdDurableChain(ctx, d, vmID) + if chain == nil { + delta = false + } else { + held := chain.base + defer func() { fm.releaseDurableChain(d, vmID, held != "" && chain.base == "") }() + } + } else if delta { // held for the whole migrate so the checkpoint loop cannot take a live // snapshot of this vm in between, which would break the diff's parent. chain = fm.chainFor(vmID) @@ -220,7 +230,7 @@ func (fm *FleetManager) migrateVM(ctx context.Context, vmID string, opts Migrate } else { // a live seed is the bulk of what moves, so the guest gives back what // it is not using first. - seed, err = fm.CreateSnapshot(ctx, vmID, SnapshotOptions{Comment: "migration seed", Live: opts.Live, ShrinkMemory: opts.Live}) + seed, err = fm.snapshotInProcess(ctx, vmID, SnapshotOptions{Comment: "migration seed", Live: opts.Live, ShrinkMemory: opts.Live}) if err != nil { return "", err } @@ -279,7 +289,7 @@ func (fm *FleetManager) migrateVM(ctx context.Context, vmID string, opts Migrate } if err == nil && delta { var child SnapshotRecord - child, err = fm.CreateSnapshot(ctx, vmID, SnapshotOptions{ + child, err = fm.snapshotInProcess(ctx, vmID, SnapshotOptions{ Comment: "migration delta", Live: true, Diff: true, Parent: seed.SnapshotID, KeepPaused: true, NoLineage: true, }) diff --git a/internal/orchestrator/snapshot_durable.go b/internal/orchestrator/snapshot_durable.go new file mode 100644 index 0000000..bd10f67 --- /dev/null +++ b/internal/orchestrator/snapshot_durable.go @@ -0,0 +1,192 @@ +package orchestrator + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + "time" +) + +var ( + // errSnapshotOptions is a snapshot asked for with options that do not go + // together, such as diff without live. + errSnapshotOptions = errors.New("invalid snapshot options") + // errNoEnvironment is a vm the fleet tracks with no environment handle. + errNoEnvironment = errors.New("vm has no active environment handle") +) + +// SnapshotFleetErrors are the sentinel errors a snapshot or restore step +// fails with when a retry cannot change the outcome. the durable handlers +// carry each one across restate as its own error code, so the api still maps +// it to the same status. +var SnapshotFleetErrors = []error{ + ErrVMNotFound, + ErrSnapshotNotFound, + ErrVMNotRunning, + ErrSnapshotInvalidState, + ErrSnapshotQuotaExceeded, + ErrGPUUnsupported, + ErrSnapshotUnsupported, +} + +// PermanentSnapshotError reports whether a snapshot or restore step failed in +// a way a retry cannot change: one of SnapshotFleetErrors, options that do not +// go together, a vm with no environment, or an agent that refused the request +// with a 4xx other than a timeout or rate limit. +func PermanentSnapshotError(err error) bool { + if err == nil { + return false + } + for _, s := range SnapshotFleetErrors { + if errors.Is(err, s) { + return true + } + } + if errors.Is(err, errSnapshotOptions) || errors.Is(err, errNoEnvironment) { + return true + } + var statusErr *HTTPStatusError + if errors.As(err, &statusErr) { + code := statusErr.Code + return code >= 400 && code < 500 && code != http.StatusRequestTimeout && code != http.StatusTooManyRequests + } + return false +} + +// DurableSnapshots runs snapshots, restores, the checkpoint chain and +// snapshot gc on the restate Environment/{vm_id} object, so each one is +// serialized with the vm's other lifecycle ops and resumes after a restart. +// *durable.Client satisfies it. +type DurableSnapshots interface { + // Snapshot runs Environment/{vmID}.Snapshot and waits for it. the + // snapshot keeps running if ctx ends first. + Snapshot(ctx context.Context, vmID string, opts SnapshotOptions) (SnapshotRecord, error) + // Restore runs Environment/{vmID}.Restore and waits for it. + Restore(ctx context.Context, vmID, snapshotID string) error + // Checkpoint advances vmID's checkpoint chain by one tick and waits + // until the tick has taken its checkpoint; the copy to the standby + // runs on after it returns. + Checkpoint(ctx context.Context, vmID string) error + // HoldChain keeps the checkpoint loop off vmID's chain for lease, or + // until ReleaseChain, and returns the chain. Busy is set while a tick's + // copy is still running; nothing is held then. + HoldChain(ctx context.Context, vmID string, lease time.Duration) (ChainHold, error) + // ReleaseChain ends a hold. reset forgets the chain, whose base the + // holder consumed or put out of step. + ReleaseChain(ctx context.Context, vmID string, reset bool) error + // SendRetireChain enqueues the removal of what a gone vm's chain holds. + SendRetireChain(ctx context.Context, vmID string) error + // SendCollectSnapshot enqueues the deletion of a snapshot retention gc + // marked deleting. key dedupes repeat sends for the same marking. + SendCollectSnapshot(ctx context.Context, vmID, snapshotID, key string) error +} + +// SetDurableSnapshots routes snapshots, restores, the checkpoint chain and +// retention gc through restate. nil (mode off, the default) keeps them in +// process. set it before Start. +func (fm *FleetManager) SetDurableSnapshots(d DurableSnapshots) { + fm.mu.Lock() + fm.durableSnapshots = d + fm.mu.Unlock() +} + +func (fm *FleetManager) durableSnapshotsClient() DurableSnapshots { + fm.mu.RLock() + defer fm.mu.RUnlock() + return fm.durableSnapshots +} + +// durableWait bounds a wait on a durable op for a caller with no deadline of +// its own (an approved hitl run), so a restate that accepts and never answers +// does not hold it forever. the op keeps running if the wait ends. +func durableWait(ctx context.Context) (context.Context, context.CancelFunc) { + if _, ok := ctx.Deadline(); ok { + return ctx, func() {} + } + return context.WithTimeout(ctx, durableMoveWait) +} + +// snapshotPinned reports whether something holds id against gc: a transfer +// in this process, or a durable pin (a move, a checkpoint chain's base). a +// durable pin that cannot be read counts as held. +func (fm *FleetManager) snapshotPinned(ctx context.Context, id string) bool { + fm.mu.RLock() + pulling := fm.artifactPulls[id] > 0 + fm.mu.RUnlock() + if pulling { + return true + } + d := fm.durableArtifactsClient() + if d == nil { + return false + } + pinned, err := d.ArtifactPinned(ctx, id) + if err != nil { + fm.logger.Warn("read durable artifact pin failed; not collecting", "snapshot", id, "err", err) + return true + } + return pinned +} + +// sendCollectSnapshot hands a snapshot retention gc marked deleting to +// restate, which retries its deletion under the Environment policy. the key +// is the marking's own time, so every sweep that finds the same marking sends +// the same key and restate runs it once. +func (fm *FleetManager) sendCollectSnapshot(ctx context.Context, d DurableSnapshots, snapshot SnapshotRecord) { + key := "collect:" + snapshot.SnapshotID + ":" + strconv.FormatInt(snapshot.UpdatedAt.UnixNano(), 10) + sctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if err := d.SendCollectSnapshot(sctx, snapshot.VMID, snapshot.SnapshotID, key); err != nil { + fm.logger.Warn("send snapshot gc failed, the next sweep retries", "snapshot", snapshot.SnapshotID, "err", err) + } +} + +// the methods below are the steps of durable snapshot gc. + +// SnapshotCollect deletes a snapshot retention gc marked deleting: its host +// artifact, then its row. a row that is gone or no longer deleting was dealt +// with already. a host that no longer has the artifact is not an error. +func (fm *FleetManager) SnapshotCollect(ctx context.Context, snapshotID string) error { + record, found, err := fm.lookupSnapshotRecord(ctx, snapshotID) + if err != nil { + return err + } + if !found || record.State != SnapshotStateDeleting { + return nil + } + if err := fm.deleteSnapshotArtifact(ctx, record); err != nil && !IsNotFound(err) { + return err + } + if fm.store != nil { + if err := fm.store.DeleteSnapshot(ctx, snapshotID); err != nil { + return fmt.Errorf("delete snapshot %s metadata: %w", snapshotID, err) + } + } + fm.appendEvent(ctx, "vm", record.VMID, "snapshot.gc_deleted", map[string]any{"snapshot_id": snapshotID}) + return nil +} + +// SnapshotCollectFailed records a gc deletion that ran out of retries: the +// row moves to error with the last error, and a later sweep marks it again. +func (fm *FleetManager) SnapshotCollectFailed(ctx context.Context, snapshotID, lastErr string) error { + record, found, err := fm.lookupSnapshotRecord(ctx, snapshotID) + if err != nil { + return err + } + if !found || record.State != SnapshotStateDeleting { + return nil + } + record.State = SnapshotStateError + record.LastError = lastErr + record.UpdatedAt = time.Now() + if err := fm.upsertSnapshotRecord(ctx, record); err != nil { + return fmt.Errorf("mark snapshot %s error after a failed gc: %w", snapshotID, err) + } + fm.appendEvent(ctx, "vm", record.VMID, "snapshot.gc_failed", map[string]any{ + "snapshot_id": snapshotID, + "error": lastErr, + }) + return nil +} diff --git a/internal/orchestrator/snapshots.go b/internal/orchestrator/snapshots.go index aed7dd0..406ce06 100644 --- a/internal/orchestrator/snapshots.go +++ b/internal/orchestrator/snapshots.go @@ -146,8 +146,24 @@ type snapshotCursor struct { // CreateSnapshot quiesces the given VM, invokes Environment.Checkpoint, // persists a first-class SnapshotRecord, and marks it ready once the -// provider confirms the new checkpoint exists. +// provider confirms the new checkpoint exists. on restate it runs as the +// Environment/{vm_id}.Snapshot handler; see snapshot_durable.go. func (fm *FleetManager) CreateSnapshot(ctx context.Context, vmID string, opts SnapshotOptions) (SnapshotRecord, error) { + if d := fm.durableSnapshotsClient(); d != nil { + ctx, end := fm.startOperation(ctx, "snapshot", vmID) + ctx, cancel := durableWait(ctx) + defer cancel() + rec, err := d.Snapshot(ctx, vmID, opts) + end(err) + return rec, err + } + return fm.snapshotInProcess(ctx, vmID, opts) +} + +// snapshotInProcess takes a snapshot in this process whatever the restate +// mode: what fork and migrate use for their seeds, and the checkpoint chain +// in mode off. +func (fm *FleetManager) snapshotInProcess(ctx context.Context, vmID string, opts SnapshotOptions) (SnapshotRecord, error) { ctx, end := fm.startOperation(ctx, "snapshot", vmID) rec, err := fm.createSnapshot(ctx, vmID, opts) end(err) @@ -155,11 +171,53 @@ func (fm *FleetManager) CreateSnapshot(ctx context.Context, vmID string, opts Sn } func (fm *FleetManager) createSnapshot(ctx context.Context, vmID string, opts SnapshotOptions) (SnapshotRecord, error) { + plan, err := fm.SnapshotBegin(ctx, vmID, opts) + if err != nil { + return SnapshotRecord{}, err + } + created, err := fm.SnapshotCheckpoint(ctx, vmID, plan) + if err != nil { + return SnapshotRecord{}, err + } + record, err := fm.SnapshotRecordCreating(ctx, vmID, plan, created) + if err != nil { + return SnapshotRecord{}, err + } + listed, err := fm.SnapshotList(ctx, vmID, record.SnapshotID) + if err != nil { + return SnapshotRecord{}, err + } + return fm.SnapshotReady(ctx, record, listed) +} + +// SnapshotPlan is what SnapshotBegin settled before the host checkpoint: +// everything the record takes that does not come back from the host. +type SnapshotPlan struct { + Options SnapshotOptions `json:"options"` + TaskID string `json:"task_id,omitempty"` + HostID string `json:"host_id,omitempty"` + TenantID string `json:"tenant_id"` + ParentSnapshotID string `json:"parent_snapshot_id,omitempty"` + Arch string `json:"arch,omitempty"` + Mode SnapshotMode `json:"mode"` + RetentionUntil *time.Time `json:"retention_until,omitempty"` + Metadata json.RawMessage `json:"metadata"` + StartedAt time.Time `json:"started_at"` +} + +// the methods below are the steps of a snapshot, in order. the durable +// Environment/{vm_id}.Snapshot handler runs each inside a journaled restate +// step; createSnapshot calls them directly. each must be safe to run again. + +// SnapshotBegin checks the vm can be snapshotted as asked and settles the +// record's fields: tenant, quota, lineage parent and metadata. it writes +// nothing. +func (fm *FleetManager) SnapshotBegin(ctx context.Context, vmID string, opts SnapshotOptions) (SnapshotPlan, error) { fm.mu.RLock() v, ok := fm.vms[vmID] if !ok { fm.mu.RUnlock() - return SnapshotRecord{}, fmt.Errorf("%w: %s", ErrVMNotFound, vmID) + return SnapshotPlan{}, fmt.Errorf("%w: %s", ErrVMNotFound, vmID) } env := v.env state := v.state @@ -192,10 +250,13 @@ func (fm *FleetManager) createSnapshot(ctx context.Context, vmID string, opts Sn // means first deciding whether pause is a fleet-visible state at all. // Behaviour is unchanged here; the question is recorded, not settled. if state != VMStateRunning { - return SnapshotRecord{}, fmt.Errorf("%w: vm %s in state %s: snapshots require running", ErrVMNotRunning, vmID, state) + return SnapshotPlan{}, fmt.Errorf("%w: vm %s in state %s: snapshots require running", ErrVMNotRunning, vmID, state) } if env == nil { - return SnapshotRecord{}, fmt.Errorf("vm %s has no active environment handle", vmID) + return SnapshotPlan{}, fmt.Errorf("%w: vm %s", errNoEnvironment, vmID) + } + if err := snapshotCapability(env, vmID, gpus, opts); err != nil { + return SnapshotPlan{}, err } now := time.Now() @@ -232,12 +293,12 @@ func (fm *FleetManager) createSnapshot(ctx context.Context, vmID string, opts Sn // tenant that starts taking live snapshots will hit an unchanged limit far // sooner than before. Those limits need re-tuning alongside enabling this. if err := fm.enforceSnapshotQuota(ctx, tenantID); err != nil { - return SnapshotRecord{}, err + return SnapshotPlan{}, err } vmSnapshots, err := fm.loadSnapshotsByVM(ctx, vmID) if err != nil { - return SnapshotRecord{}, err + return SnapshotPlan{}, err } parentSnapshotID := latestReadySnapshotID(vmSnapshots, vmID) if opts.NoLineage { @@ -245,136 +306,227 @@ func (fm *FleetManager) createSnapshot(ctx context.Context, vmID string, opts Sn } metadataJSON, err := marshalSnapshotMetadata(opts.Comment, opts.Metadata) if err != nil { - return SnapshotRecord{}, fmt.Errorf("marshal snapshot metadata: %w", err) + return SnapshotPlan{}, fmt.Errorf("marshal snapshot metadata: %w", err) } + return SnapshotPlan{ + Options: opts, + TaskID: taskID, + HostID: hostID, + TenantID: tenantID, + ParentSnapshotID: parentSnapshotID, + Arch: arch, + Mode: mode, + RetentionUntil: retentionUntil, + Metadata: metadataJSON, + StartedAt: now, + }, nil +} - sc, ok := env.(SnapshotCapable) - if !ok { +// snapshotCapability says why env cannot take the snapshot opts asks for, +// or nil if it can. +func snapshotCapability(env Environment, vmID string, gpus int32, opts SnapshotOptions) error { + if _, ok := env.(SnapshotCapable); !ok { // gpu environments run under a backend that passes the device through // via vfio, which cannot be checkpointed (d4). the env deliberately // omits SnapshotCapable; surface that reason rather than a generic one. if gpus > 0 { - return SnapshotRecord{}, fmt.Errorf("%w: vm %s has a gpu passthrough device: snapshots are not supported for gpu environments", ErrGPUUnsupported, vmID) + return fmt.Errorf("%w: vm %s has a gpu passthrough device: snapshots are not supported for gpu environments", ErrGPUUnsupported, vmID) } - return SnapshotRecord{}, fmt.Errorf("%w: provider does not support snapshots for vm %s", ErrSnapshotUnsupported, vmID) + return fmt.Errorf("%w: provider does not support snapshots for vm %s", ErrSnapshotUnsupported, vmID) } // The live path is a second, narrower capability rather than a flag on // Checkpoint, and it is asserted AFTER the SnapshotCapable block above so // a gpu env still fails with the gpu explanation: a gpu env implements // neither interface, and "your gpu cannot be checkpointed at all" is the // useful answer to `--live` on one, not "live snapshots are unsupported". - var created Checkpoint - if opts.Diff && opts.Live { - dc, ok := env.(DeltaSnapshotCapable) - if !ok { - return SnapshotRecord{}, fmt.Errorf("%w: provider does not support diff snapshots for vm %s", ErrSnapshotUnsupported, vmID) + switch { + case opts.Diff && !opts.Live: + return fmt.Errorf("%w: diff snapshot of vm %s requires live", errSnapshotOptions, vmID) + case opts.Diff: + if _, ok := env.(DeltaSnapshotCapable); !ok { + return fmt.Errorf("%w: provider does not support diff snapshots for vm %s", ErrSnapshotUnsupported, vmID) } - created, err = dc.CheckpointDiff(ctx, opts.Comment, opts.Parent, opts.KeepPaused) - if err != nil { - return SnapshotRecord{}, fmt.Errorf("diff checkpoint %s: %w", vmID, err) + case opts.Live: + if _, ok := env.(LiveSnapshotCapable); !ok { + return fmt.Errorf("%w: provider does not support live snapshots for vm %s: retry without live to take a disk snapshot", ErrSnapshotUnsupported, vmID) } - } else if opts.Diff { - return SnapshotRecord{}, fmt.Errorf("diff snapshot of vm %s requires live", vmID) - } else if opts.Live { - lc, ok := env.(LiveSnapshotCapable) - if !ok { - return SnapshotRecord{}, fmt.Errorf("%w: provider does not support live snapshots for vm %s: retry without live to take a disk snapshot", ErrSnapshotUnsupported, vmID) + } + return nil +} + +// SnapshotCheckpoint takes the checkpoint on the vm's host. on restate the +// call carries the step's idempotency key, so a retry after a lost answer +// gets the first checkpoint back instead of a second one. +func (fm *FleetManager) SnapshotCheckpoint(ctx context.Context, vmID string, plan SnapshotPlan) (Checkpoint, error) { + fm.mu.RLock() + v, ok := fm.vms[vmID] + var env Environment + var gpus int32 + if ok { + env, gpus = v.env, v.spec.GPUs + } + fm.mu.RUnlock() + if !ok { + return Checkpoint{}, fmt.Errorf("%w: %s", ErrVMNotFound, vmID) + } + if env == nil { + return Checkpoint{}, fmt.Errorf("%w: vm %s", errNoEnvironment, vmID) + } + opts := plan.Options + if err := snapshotCapability(env, vmID, gpus, opts); err != nil { + return Checkpoint{}, err + } + + var created Checkpoint + var err error + switch { + case opts.Diff: + created, err = env.(DeltaSnapshotCapable).CheckpointDiff(ctx, opts.Comment, opts.Parent, opts.KeepPaused) + if err != nil { + return Checkpoint{}, fmt.Errorf("diff checkpoint %s: %w", vmID, err) } + case opts.Live: if mc, ok := env.(MemoryShrinkCapable); ok && opts.ShrinkMemory { created, err = mc.CheckpointLiveShrunk(ctx, opts.Comment) } else { - created, err = lc.CheckpointLive(ctx, opts.Comment) + created, err = env.(LiveSnapshotCapable).CheckpointLive(ctx, opts.Comment) } if err != nil { - return SnapshotRecord{}, fmt.Errorf("live checkpoint %s: %w", vmID, err) + return Checkpoint{}, fmt.Errorf("live checkpoint %s: %w", vmID, err) } - } else { - id, digest, cErr := checkpointWithDigest(ctx, sc, opts.Comment) + default: + id, digest, cErr := checkpointWithDigest(ctx, env.(SnapshotCapable), opts.Comment) if cErr != nil { - return SnapshotRecord{}, fmt.Errorf("checkpoint %s: %w", vmID, cErr) + return Checkpoint{}, fmt.Errorf("checkpoint %s: %w", vmID, cErr) } created = Checkpoint{ID: id, Digest: digest, Kind: SnapshotKindDisk} } - snapshotID, digest := created.ID, created.Digest // The kind is whatever the provider says it wrote, not what was asked // for: an agent too old to know about live snapshots answers a live // request with a disk snapshot, and recording the request instead of the // result would file a cold-booting artifact as a resumable one. - kind := created.Kind - if kind == "" { - kind = SnapshotKindDisk + if created.Kind == "" { + created.Kind = SnapshotKindDisk } + return created, nil +} + +// SnapshotRecordCreating persists the record of a checkpoint the host just +// took, in state creating. +func (fm *FleetManager) SnapshotRecordCreating(ctx context.Context, vmID string, plan SnapshotPlan, created Checkpoint) (SnapshotRecord, error) { + metadataJSON := plan.Metadata if len(created.Files) > 0 { // same reasoning as the digest: this response is the only moment the // per-file digests exist, and a live snapshot recorded without them can // never be verified on another host, so it can never leave this one. - metadata := withLiveFiles(opts.Metadata, created.Files) + metadata := withLiveFiles(plan.Options.Metadata, created.Files) if created.Parent != "" { metadata[diffParentMetadataKey] = created.Parent } - metadataJSON, err = marshalSnapshotMetadata(opts.Comment, metadata) + var err error + metadataJSON, err = marshalSnapshotMetadata(plan.Options.Comment, metadata) if err != nil { return SnapshotRecord{}, fmt.Errorf("marshal snapshot metadata: %w", err) } } - checkpoint, _ := lookupCheckpoint(ctx, env, snapshotID) - sizeBytes := checkpoint.SizeBytes - if sizeBytes == 0 { - // The listing stays the primary source so nothing about existing disk - // snapshot accounting moves. The create response only fills the gap it - // leaves: an agent that lists snapshots without sizes would otherwise - // record a live artifact, memory image and all, as zero bytes and hand - // the quota a number it knows to be wrong. - sizeBytes = created.SizeBytes - } - - createdAt := now - if !checkpoint.CreatedAt.IsZero() { - createdAt = checkpoint.CreatedAt - } record := SnapshotRecord{ - SnapshotID: snapshotID, + SnapshotID: created.ID, VMID: vmID, - TaskID: taskID, - HostID: hostID, - TenantID: tenantID, - ParentSnapshotID: parentSnapshotID, - Mode: mode, - LayerKey: opts.LayerKey, - Arch: arch, - Digest: digest, - Kind: kind, + TaskID: plan.TaskID, + HostID: plan.HostID, + TenantID: plan.TenantID, + ParentSnapshotID: plan.ParentSnapshotID, + Mode: plan.Mode, + LayerKey: plan.Options.LayerKey, + Arch: plan.Arch, + Digest: created.Digest, + Kind: created.Kind, State: SnapshotStateCreating, - SizeBytes: sizeBytes, - RetentionUntil: retentionUntil, + SizeBytes: created.SizeBytes, + RetentionUntil: plan.RetentionUntil, Metadata: metadataJSON, - Exports: cloneSnapshotExports(opts.Exports), - CreatedAt: createdAt, - UpdatedAt: now, + Exports: cloneSnapshotExports(plan.Options.Exports), + CreatedAt: plan.StartedAt, + UpdatedAt: time.Now(), } - if err := fm.upsertSnapshotRecord(ctx, record); err != nil { - return SnapshotRecord{}, fmt.Errorf("persist snapshot %s: %w", snapshotID, err) + return SnapshotRecord{}, fmt.Errorf("persist snapshot %s: %w", created.ID, err) } + return record, nil +} +// SnapshotList reads the new checkpoint back from the host's listing, for +// its size and creation time. a listing that fails or leaves it out is not +// an error: the record keeps what the create response said. +func (fm *FleetManager) SnapshotList(ctx context.Context, vmID, snapshotID string) (Checkpoint, error) { + fm.mu.RLock() + var env Environment + if v, ok := fm.vms[vmID]; ok { + env = v.env + } + fm.mu.RUnlock() + if env == nil { + return Checkpoint{}, nil + } + checkpoint, _ := lookupCheckpoint(ctx, env, snapshotID) + return checkpoint, nil +} + +// SnapshotReady marks the record ready with what the listing reported. +func (fm *FleetManager) SnapshotReady(ctx context.Context, record SnapshotRecord, listed Checkpoint) (SnapshotRecord, error) { + // The listing stays the primary source so nothing about existing disk + // snapshot accounting moves. The create response only fills the gap it + // leaves: an agent that lists snapshots without sizes would otherwise + // record a live artifact, memory image and all, as zero bytes and hand + // the quota a number it knows to be wrong. + if listed.SizeBytes != 0 { + record.SizeBytes = listed.SizeBytes + } + if !listed.CreatedAt.IsZero() { + record.CreatedAt = listed.CreatedAt + } record.State = SnapshotStateReady record.LastError = "" record.UpdatedAt = time.Now() if err := fm.upsertSnapshotRecord(ctx, record); err != nil { - return SnapshotRecord{}, fmt.Errorf("mark snapshot %s ready: %w", snapshotID, err) + return SnapshotRecord{}, fmt.Errorf("mark snapshot %s ready: %w", record.SnapshotID, err) } - fm.appendEvent(ctx, "vm", vmID, "snapshot.created", map[string]any{ - "snapshot_id": snapshotID, - "mode": string(mode), - "parent_snapshot_id": parentSnapshotID, - "size_bytes": sizeBytes, - "kind": string(kind), + fm.appendEvent(ctx, "vm", record.VMID, "snapshot.created", map[string]any{ + "snapshot_id": record.SnapshotID, + "mode": string(record.Mode), + "parent_snapshot_id": record.ParentSnapshotID, + "size_bytes": record.SizeBytes, + "kind": string(record.Kind), }) - return record, nil } +// SnapshotAbort is the compensation of a snapshot whose checkpoint the host +// took but whose record could not be finished: the host artifact is deleted +// and the row, if one was written, is marked error with the cause. +func (fm *FleetManager) SnapshotAbort(ctx context.Context, vmID string, plan SnapshotPlan, created Checkpoint, cause string) error { + record := SnapshotRecord{SnapshotID: created.ID, VMID: vmID, HostID: plan.HostID} + if err := fm.deleteSnapshotArtifact(ctx, record); err != nil && !IsNotFound(err) { + return err + } + existing, found, err := fm.lookupSnapshotRecord(ctx, created.ID) + if err != nil { + return err + } + if found { + existing.State = SnapshotStateError + existing.LastError = cause + existing.UpdatedAt = time.Now() + if err := fm.upsertSnapshotRecord(ctx, existing); err != nil { + return fmt.Errorf("mark snapshot %s error: %w", created.ID, err) + } + } + fm.appendEvent(ctx, "vm", vmID, "snapshot.failed", map[string]any{"snapshot_id": created.ID, "error": cause}) + fm.logger.WarnContext(ctx, "snapshot aborted, host artifact deleted", "vm", vmID, "snapshot", created.ID, "err", cause) + return nil +} + // ListSnapshots returns all snapshots known to the state store for the // given VM ID, newest first. Returns an empty slice when the VM exists // but has no snapshots. Scoped to one VM's snapshots, so it is not the @@ -566,20 +718,50 @@ func (fm *FleetManager) deleteSnapshotRecord(ctx context.Context, record Snapsho // RestoreSnapshot rolls a VM back to a prior snapshot via // Environment.Restore after validating the metadata record and provider -// visibility of the checkpoint. +// visibility of the checkpoint. on restate it runs as the +// Environment/{vm_id}.Restore handler, so a restart resumes it. func (fm *FleetManager) RestoreSnapshot(ctx context.Context, vmID, snapshotID string) error { ctx, end := fm.startOperation(ctx, "restore", vmID) - err := fm.restoreSnapshot(ctx, vmID, snapshotID) + var err error + if d := fm.durableSnapshotsClient(); d != nil { + wctx, cancel := durableWait(ctx) + err = d.Restore(wctx, vmID, snapshotID) + cancel() + } else { + err = fm.restoreSnapshot(ctx, vmID, snapshotID) + } end(err) return err } func (fm *FleetManager) restoreSnapshot(ctx context.Context, vmID, snapshotID string) error { + record, err := fm.RestoreValidate(ctx, vmID, snapshotID) + if err != nil { + return err + } + if err := fm.RestoreBegin(ctx, record); err != nil { + return err + } + if err := fm.RestoreOnHost(ctx, vmID, snapshotID); err != nil { + _ = fm.RestoreFailed(ctx, record, err.Error()) + return err + } + return fm.RestoreFinish(ctx, record) +} + +// the methods below are the steps of a restore, in order. the durable +// Environment/{vm_id}.Restore handler runs each inside a journaled restate +// step; restoreSnapshot calls them directly. each must be safe to run again. + +// RestoreValidate checks the vm can be restored to snapshotID and that its +// host still holds the checkpoint, and returns the record. a checkpoint the +// host no longer has marks the record error. +func (fm *FleetManager) RestoreValidate(ctx context.Context, vmID, snapshotID string) (SnapshotRecord, error) { fm.mu.RLock() v, ok := fm.vms[vmID] if !ok { fm.mu.RUnlock() - return fmt.Errorf("%w: %s", ErrVMNotFound, vmID) + return SnapshotRecord{}, fmt.Errorf("%w: %s", ErrVMNotFound, vmID) } env := v.env state := v.state @@ -592,64 +774,86 @@ func (fm *FleetManager) restoreSnapshot(ctx context.Context, vmID, snapshotID st // Left as is deliberately, for the same reason: there is no fleet-visible // paused state to admit. if state != VMStateRunning { - return fmt.Errorf("%w: vm %s in state %s: restore requires running", ErrVMNotRunning, vmID, state) + return SnapshotRecord{}, fmt.Errorf("%w: vm %s in state %s: restore requires running", ErrVMNotRunning, vmID, state) } if env == nil { - return fmt.Errorf("vm %s has no active environment handle", vmID) + return SnapshotRecord{}, fmt.Errorf("%w: vm %s", errNoEnvironment, vmID) } record, err := fm.getSnapshotRecord(ctx, snapshotID) if err != nil { - return err + return SnapshotRecord{}, err } if record.VMID != vmID { - return fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) + return SnapshotRecord{}, fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } if record.State != SnapshotStateReady { - return fmt.Errorf("%w: snapshot %s is %s", ErrSnapshotInvalidState, snapshotID, record.State) + return SnapshotRecord{}, fmt.Errorf("%w: snapshot %s is %s", ErrSnapshotInvalidState, snapshotID, record.State) + } + if _, ok := env.(SnapshotCapable); !ok { + return SnapshotRecord{}, fmt.Errorf("%w: restore %s on vm %s: provider does not support snapshots", ErrSnapshotUnsupported, snapshotID, vmID) } if _, ok := lookupCheckpoint(ctx, env, snapshotID); !ok { record.State = SnapshotStateError record.LastError = "snapshot missing from provider" record.UpdatedAt = time.Now() _ = fm.upsertSnapshotRecord(ctx, record) - return fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) + return SnapshotRecord{}, fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } + return record, nil +} +// RestoreBegin marks the record restoring. +func (fm *FleetManager) RestoreBegin(ctx context.Context, record SnapshotRecord) error { record.State = SnapshotStateRestoring record.LastError = "" record.UpdatedAt = time.Now() if err := fm.upsertSnapshotRecord(ctx, record); err != nil { - return fmt.Errorf("mark snapshot %s restoring: %w", snapshotID, err) + return fmt.Errorf("mark snapshot %s restoring: %w", record.SnapshotID, err) } + return nil +} +// RestoreOnHost has the vm's host restore the checkpoint. +func (fm *FleetManager) RestoreOnHost(ctx context.Context, vmID, snapshotID string) error { + fm.mu.RLock() + var env Environment + if v, ok := fm.vms[vmID]; ok { + env = v.env + } + fm.mu.RUnlock() sc, ok := env.(SnapshotCapable) if !ok { - record.State = SnapshotStateError - record.LastError = "provider does not support snapshots" - record.UpdatedAt = time.Now() - _ = fm.upsertSnapshotRecord(ctx, record) - return fmt.Errorf("%w: restore %s on vm %s: provider does not support snapshots", ErrSnapshotUnsupported, snapshotID, vmID) + return fmt.Errorf("%w: restore %s on vm %s: no environment that can restore it", ErrSnapshotUnsupported, snapshotID, vmID) } if err := sc.Restore(ctx, snapshotID); err != nil { - record.State = SnapshotStateError - record.LastError = err.Error() - record.UpdatedAt = time.Now() - _ = fm.upsertSnapshotRecord(ctx, record) return fmt.Errorf("restore %s on vm %s: %w", snapshotID, vmID, err) } + return nil +} +// RestoreFinish marks the record ready again once the vm runs from it. +func (fm *FleetManager) RestoreFinish(ctx context.Context, record SnapshotRecord) error { record.State = SnapshotStateReady record.LastError = "" record.UpdatedAt = time.Now() if err := fm.upsertSnapshotRecord(ctx, record); err != nil { - return fmt.Errorf("mark snapshot %s ready after restore: %w", snapshotID, err) + return fmt.Errorf("mark snapshot %s ready after restore: %w", record.SnapshotID, err) } - - fm.appendEvent(ctx, "vm", vmID, "snapshot.restored", map[string]any{ - "snapshot_id": snapshotID, + fm.appendEvent(ctx, "vm", record.VMID, "snapshot.restored", map[string]any{ + "snapshot_id": record.SnapshotID, }) + return nil +} +// RestoreFailed marks the record error with the restore's last error. +func (fm *FleetManager) RestoreFailed(ctx context.Context, record SnapshotRecord, lastErr string) error { + record.State = SnapshotStateError + record.LastError = lastErr + record.UpdatedAt = time.Now() + if err := fm.upsertSnapshotRecord(ctx, record); err != nil { + return fmt.Errorf("mark snapshot %s error after a failed restore: %w", record.SnapshotID, err) + } return nil } @@ -673,6 +877,7 @@ func (fm *FleetManager) reconcileSnapshots(ctx context.Context) { } now := time.Now() + d := fm.durableSnapshotsClient() children := make(map[string]int, len(all)) latestPerVM := make(map[string]SnapshotRecord) for _, snapshot := range all { @@ -701,6 +906,11 @@ func (fm *FleetManager) reconcileSnapshots(ctx context.Context) { if latest, ok := latestPerVM[snapshot.VMID]; ok && latest.SnapshotID == snapshot.SnapshotID { continue } + // a checkpoint a move is still reading, or a chain's base, stays + // until its pin is released. + if fm.snapshotPinned(ctx, snapshot.SnapshotID) { + continue + } snapshot.State = SnapshotStateDeleting snapshot.UpdatedAt = now snapshot.LastError = "" @@ -710,6 +920,12 @@ func (fm *FleetManager) reconcileSnapshots(ctx context.Context) { } } + if d != nil && snapshot.VMID != "" { + // restate retries the deletion under the Environment policy, and + // only marks the row error once that runs out. + fm.sendCollectSnapshot(ctx, d, snapshot) + continue + } if err := fm.deleteSnapshotArtifact(ctx, snapshot); err != nil { snapshot.State = SnapshotStateError snapshot.LastError = err.Error() diff --git a/orchestrator/main.go b/orchestrator/main.go index 3726e76..c1366b2 100644 --- a/orchestrator/main.go +++ b/orchestrator/main.go @@ -571,6 +571,7 @@ func run() error { restateClient = durable.NewClient(durableCfg) fm.SetDurableDestroyer(restateClient) fm.SetDurableArtifacts(restateClient) + fm.SetDurableSnapshots(restateClient) } fm.Start(rootCtx) defer fm.Stop() From d4b974fc38f309bcc19543673aeee1e3a1a393e0 Mon Sep 17 00:00:00 2001 From: andrewn6 <59238070+andrewn6@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:33:41 -0500 Subject: [PATCH 3/6] test: durable snapshot, restore and checkpoint chain, including endpoint restarts mid-snapshot and mid-restore --- internal/durable/server_test.go | 48 +- internal/durable/snapshot_test.go | 521 ++++++++++++++++++ .../orchestrator/checkpoints_durable_test.go | 145 +++++ .../orchestrator/snapshot_durable_test.go | 295 ++++++++++ 4 files changed, 1008 insertions(+), 1 deletion(-) create mode 100644 internal/durable/snapshot_test.go create mode 100644 internal/orchestrator/checkpoints_durable_test.go create mode 100644 internal/orchestrator/snapshot_durable_test.go diff --git a/internal/durable/server_test.go b/internal/durable/server_test.go index 1ea80a7..9f1b2bd 100644 --- a/internal/durable/server_test.go +++ b/internal/durable/server_test.go @@ -55,6 +55,50 @@ func (fakeFleet) ArtifactRecord(_ context.Context, job orchestrator.ArtifactMove func (fakeFleet) ArtifactDiscard(context.Context, orchestrator.ArtifactMoveJob) error { return nil } func (fakeFleet) ArtifactPullTimeout() time.Duration { return 0 } +func (fakeFleet) SnapshotBegin(_ context.Context, _ string, opts orchestrator.SnapshotOptions) (orchestrator.SnapshotPlan, error) { + return orchestrator.SnapshotPlan{Options: opts}, nil +} +func (fakeFleet) SnapshotCheckpoint(context.Context, string, orchestrator.SnapshotPlan) (orchestrator.Checkpoint, error) { + return orchestrator.Checkpoint{ID: "snap-1"}, nil +} +func (fakeFleet) SnapshotRecordCreating(_ context.Context, vm string, _ orchestrator.SnapshotPlan, cp orchestrator.Checkpoint) (orchestrator.SnapshotRecord, error) { + return orchestrator.SnapshotRecord{SnapshotID: cp.ID, VMID: vm, State: orchestrator.SnapshotStateCreating}, nil +} +func (fakeFleet) SnapshotList(context.Context, string, string) (orchestrator.Checkpoint, error) { + return orchestrator.Checkpoint{}, nil +} +func (fakeFleet) SnapshotReady(_ context.Context, rec orchestrator.SnapshotRecord, _ orchestrator.Checkpoint) (orchestrator.SnapshotRecord, error) { + rec.State = orchestrator.SnapshotStateReady + return rec, nil +} +func (fakeFleet) SnapshotAbort(context.Context, string, orchestrator.SnapshotPlan, orchestrator.Checkpoint, string) error { + return nil +} +func (fakeFleet) RestoreValidate(_ context.Context, vm, id string) (orchestrator.SnapshotRecord, error) { + return orchestrator.SnapshotRecord{SnapshotID: id, VMID: vm}, nil +} +func (fakeFleet) RestoreBegin(context.Context, orchestrator.SnapshotRecord) error { return nil } +func (fakeFleet) RestoreOnHost(context.Context, string, string) error { return nil } +func (fakeFleet) RestoreFinish(context.Context, orchestrator.SnapshotRecord) error { return nil } +func (fakeFleet) RestoreFailed(context.Context, orchestrator.SnapshotRecord, string) error { + return nil +} +func (fakeFleet) SnapshotCollect(context.Context, string) error { return nil } +func (fakeFleet) SnapshotCollectFailed(context.Context, string, string) error { return nil } + +func (fakeFleet) ChainBegin(context.Context, string, orchestrator.ChainState) (orchestrator.ChainPlan, error) { + return orchestrator.ChainPlan{}, nil +} +func (fakeFleet) ChainMovePlan(context.Context, orchestrator.ChainState, orchestrator.ChainPlan, orchestrator.SnapshotRecord) (orchestrator.ChainMove, error) { + return orchestrator.ChainMove{}, nil +} +func (fakeFleet) ChainRetire(context.Context, string, orchestrator.ChainState) error { return nil } +func (fakeFleet) ChainReset(context.Context, string, string) error { return nil } +func (fakeFleet) ChainAdvanced(context.Context, string, orchestrator.ChainState, bool) error { + return nil +} +func (fakeFleet) MirrorChain(string, orchestrator.ChainState, bool) {} + var testLogger = slog.New(slog.NewTextHandler(io.Discard, nil)) // discovery is the manifest restate-server reads on registration. checking it @@ -113,7 +157,9 @@ func TestDiscoveryManifest(t *testing.T) { want := map[string]string{"Ping": "SHARED"} switch s.Name { case EnvironmentObject: - want["Destroy"] = "EXCLUSIVE" + for _, h := range []string{"Destroy", "Snapshot", "Restore", "CollectSnapshot", "Checkpoint", "ChainMoved", "HoldChain", "ReleaseChain", "RetireChain"} { + want[h] = "EXCLUSIVE" + } case ArtifactsObject: want = map[string]string{"Move": "EXCLUSIVE"} case ArtifactPinObject: diff --git a/internal/durable/snapshot_test.go b/internal/durable/snapshot_test.go new file mode 100644 index 0000000..8dddfa6 --- /dev/null +++ b/internal/durable/snapshot_test.go @@ -0,0 +1,521 @@ +package durable + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/folsomintel/fuse/internal/orchestrator" +) + +// snapFleet scripts the snapshot, restore, gc and checkpoint chain steps. +// fail[step] errors are returned in turn before the step succeeds; block, +// when set for a step, holds it until closed and signals entered first. +type snapFleet struct { + fakeFleet + + mu sync.Mutex + calls []string + fail map[string][]error + block map[string]chan struct{} + entered chan string + seq int + aborted string + failed string + retired []orchestrator.ChainState + mirror orchestrator.ChainState + pending bool + mirrors int +} + +func newSnapFleet() *snapFleet { + return &snapFleet{fail: map[string][]error{}, block: map[string]chan struct{}{}, entered: make(chan string, 16)} +} + +// enter records step, waits if it is blocked, and returns its next scripted +// error. +func (f *snapFleet) enter(ctx context.Context, step string) error { + f.mu.Lock() + f.calls = append(f.calls, step) + block := f.block[step] + f.mu.Unlock() + if block != nil { + select { + case f.entered <- step: + default: + } + select { + case <-block: + case <-ctx.Done(): + return ctx.Err() + } + } + f.mu.Lock() + defer f.mu.Unlock() + if errs := f.fail[step]; len(errs) > 0 { + err := errs[0] + if len(errs) > 1 { + f.fail[step] = errs[1:] + } else if errs[0] != errAlways { + delete(f.fail, step) + } + return err + } + return nil +} + +// errAlways, as the last scripted error of a step, makes it fail forever. +var errAlways = errors.New("store down") + +func (f *snapFleet) steps() string { + f.mu.Lock() + defer f.mu.Unlock() + return strings.Join(f.calls, ",") +} + +func (f *snapFleet) SnapshotBegin(ctx context.Context, vm string, opts orchestrator.SnapshotOptions) (orchestrator.SnapshotPlan, error) { + return orchestrator.SnapshotPlan{Options: opts, HostID: "h1"}, f.enter(ctx, "begin") +} + +func (f *snapFleet) SnapshotCheckpoint(ctx context.Context, vm string, plan orchestrator.SnapshotPlan) (orchestrator.Checkpoint, error) { + if err := f.enter(ctx, "checkpoint"); err != nil { + return orchestrator.Checkpoint{}, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.seq++ + return orchestrator.Checkpoint{ID: fmt.Sprintf("snap-%d", f.seq), SizeBytes: 10, Kind: orchestrator.SnapshotKindLive}, nil +} + +func (f *snapFleet) SnapshotRecordCreating(ctx context.Context, vm string, plan orchestrator.SnapshotPlan, cp orchestrator.Checkpoint) (orchestrator.SnapshotRecord, error) { + return orchestrator.SnapshotRecord{SnapshotID: cp.ID, VMID: vm, HostID: plan.HostID, SizeBytes: cp.SizeBytes, + State: orchestrator.SnapshotStateCreating}, f.enter(ctx, "record") +} + +func (f *snapFleet) SnapshotList(ctx context.Context, vm, id string) (orchestrator.Checkpoint, error) { + return orchestrator.Checkpoint{ID: id}, f.enter(ctx, "list") +} + +func (f *snapFleet) SnapshotReady(ctx context.Context, rec orchestrator.SnapshotRecord, _ orchestrator.Checkpoint) (orchestrator.SnapshotRecord, error) { + rec.State = orchestrator.SnapshotStateReady + return rec, f.enter(ctx, "ready") +} + +func (f *snapFleet) SnapshotAbort(ctx context.Context, vm string, _ orchestrator.SnapshotPlan, cp orchestrator.Checkpoint, cause string) error { + f.mu.Lock() + f.aborted = cp.ID + ": " + cause + f.mu.Unlock() + return f.enter(ctx, "abort") +} + +func (f *snapFleet) RestoreValidate(ctx context.Context, vm, id string) (orchestrator.SnapshotRecord, error) { + return orchestrator.SnapshotRecord{SnapshotID: id, VMID: vm}, f.enter(ctx, "validate") +} + +func (f *snapFleet) RestoreBegin(ctx context.Context, _ orchestrator.SnapshotRecord) error { + return f.enter(ctx, "restoring") +} + +func (f *snapFleet) RestoreOnHost(ctx context.Context, _, _ string) error { + return f.enter(ctx, "restore") +} + +func (f *snapFleet) RestoreFinish(ctx context.Context, _ orchestrator.SnapshotRecord) error { + return f.enter(ctx, "finish") +} + +func (f *snapFleet) RestoreFailed(ctx context.Context, _ orchestrator.SnapshotRecord, lastErr string) error { + f.mu.Lock() + f.failed = lastErr + f.mu.Unlock() + return f.enter(ctx, "failed") +} + +func (f *snapFleet) SnapshotCollect(ctx context.Context, _ string) error { + return f.enter(ctx, "collect") +} + +func (f *snapFleet) SnapshotCollectFailed(ctx context.Context, _, lastErr string) error { + f.mu.Lock() + f.failed = lastErr + f.mu.Unlock() + return f.enter(ctx, "collect_failed") +} + +func (f *snapFleet) ChainBegin(ctx context.Context, vm string, chain orchestrator.ChainState) (orchestrator.ChainPlan, error) { + if err := f.enter(ctx, "plan"); err != nil { + return orchestrator.ChainPlan{}, err + } + if chain.Base != "" { + return orchestrator.ChainPlan{Diff: true, Standby: chain.Standby, + Options: orchestrator.SnapshotOptions{Live: true, Diff: true, Parent: chain.Head.SnapshotID}}, nil + } + return orchestrator.ChainPlan{Standby: "h2", Options: orchestrator.SnapshotOptions{Live: true}}, nil +} + +func (f *snapFleet) ChainMovePlan(ctx context.Context, chain orchestrator.ChainState, plan orchestrator.ChainPlan, child orchestrator.SnapshotRecord) (orchestrator.ChainMove, error) { + job := orchestrator.ArtifactMoveJob{ + SnapshotID: child.SnapshotID, SourceSnapshotID: child.SnapshotID, SourceHostID: "h1", + HostID: plan.Standby, LocalID: fmt.Sprintf("%s-on-%s-%d", child.SnapshotID, plan.Standby, time.Now().UnixNano()), + } + if plan.Diff { + job.Base, job.DropBase = chain.Base, true + } + return orchestrator.ChainMove{Job: job}, f.enter(ctx, "plan_move") +} + +func (f *snapFleet) ChainRetire(ctx context.Context, _ string, chain orchestrator.ChainState) error { + f.mu.Lock() + f.retired = append(f.retired, chain) + f.mu.Unlock() + return f.enter(ctx, "retire") +} + +func (f *snapFleet) ChainReset(ctx context.Context, _, _ string) error { + return f.enter(ctx, "reset") +} + +func (f *snapFleet) ChainAdvanced(ctx context.Context, _ string, _ orchestrator.ChainState, _ bool) error { + return f.enter(ctx, "advanced") +} + +func (f *snapFleet) MirrorChain(_ string, chain orchestrator.ChainState, pending bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.mirror, f.pending = chain, pending + f.mirrors++ +} + +func (f *snapFleet) mirrored() (orchestrator.ChainState, bool) { + f.mu.Lock() + defer f.mu.Unlock() + return f.mirror, f.pending +} + +// waitSettled polls the mirror until the chain's copy has ended. +func (f *snapFleet) waitSettled(t *testing.T) orchestrator.ChainState { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + chain, pending := f.mirrored() + if !pending { + return chain + } + if time.Now().After(deadline) { + t.Fatal("chain copy never settled") + } + time.Sleep(50 * time.Millisecond) + } +} + +func uniqueVM(name string) string { + return fmt.Sprintf("fuse-%s-%d", name, time.Now().UnixNano()) +} + +// TestSnapshotThroughRestate: a host checkpoint that fails twice is retried +// inside its step, and the record runs once against the checkpoint that +// succeeded. +func TestSnapshotThroughRestate(t *testing.T) { + fleet := newSnapFleet() + fleet.fail["checkpoint"] = []error{errors.New("agent 503"), errors.New("agent 503")} + client := serveFleet(t, fleet, fastPolicy) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + rec, err := client.Snapshot(ctx, uniqueVM("snap"), orchestrator.SnapshotOptions{Comment: "c"}) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + if rec.SnapshotID != "snap-1" || rec.State != orchestrator.SnapshotStateReady { + t.Fatalf("record %+v, want snap-1 ready", rec) + } + if got, want := fleet.steps(), "begin,checkpoint,checkpoint,checkpoint,record,list,ready"; got != want { + t.Fatalf("steps %s, want %s", got, want) + } +} + +// TestSnapshotRefusalKeepsItsSentinel: a refusal crosses restate as its code +// and comes back as the fleet error the api maps to a status. +func TestSnapshotRefusalKeepsItsSentinel(t *testing.T) { + fleet := newSnapFleet() + fleet.fail["begin"] = []error{fmt.Errorf("%w: vm in state stopped", orchestrator.ErrVMNotRunning)} + client := serveFleet(t, fleet, fastPolicy) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + _, err := client.Snapshot(ctx, uniqueVM("refused"), orchestrator.SnapshotOptions{}) + if !errors.Is(err, orchestrator.ErrVMNotRunning) { + t.Fatalf("snapshot error %v, want ErrVMNotRunning", err) + } + if got := fleet.steps(); got != "begin" { + t.Fatalf("steps %s, want only begin", got) + } +} + +// TestSnapshotAbortsWhenTheRecordCannotFinish: a ready write that never +// succeeds runs the compensation, which deletes the host artifact and marks +// the row error. +func TestSnapshotAbortsWhenTheRecordCannotFinish(t *testing.T) { + fleet := newSnapFleet() + fleet.fail["ready"] = []error{errAlways} + client := serveFleet(t, fleet, fastPolicy) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if _, err := client.Snapshot(ctx, uniqueVM("abort"), orchestrator.SnapshotOptions{}); err == nil { + t.Fatal("snapshot succeeded with a ready write that never does") + } + want := "begin,checkpoint,record,list," + strings.Repeat("ready,", fastPolicy.MaxAttempts) + "abort" + if got := fleet.steps(); got != want { + t.Fatalf("steps %s, want %s", got, want) + } + if !strings.HasPrefix(fleet.aborted, "snap-1: ") || !strings.Contains(fleet.aborted, "store down") { + t.Fatalf("abort %q, want snap-1 with the ready error", fleet.aborted) + } +} + +// TestSnapshotSurvivesEndpointRestart stands in for kill -9 of the +// orchestrator between the host checkpoint and the row write: the endpoint +// drops while the record step runs, and the restarted one writes the row for +// the checkpoint already taken instead of taking a second one. +func TestSnapshotSurvivesEndpointRestart(t *testing.T) { + first := newSnapFleet() + first.block["record"] = make(chan struct{}) + defer close(first.block["record"]) + client, addr, stop := serveFleetAt(t, first, "") + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + done := make(chan error, 1) + var rec orchestrator.SnapshotRecord + go func() { + var err error + rec, err = client.Snapshot(ctx, uniqueVM("snap-restart"), orchestrator.SnapshotOptions{}) + done <- err + }() + <-first.entered + stop() + + second := newSnapFleet() + _, _, _ = serveFleetAt(t, second, addr) + if err := <-done; err != nil { + t.Fatalf("snapshot: %v", err) + } + if rec.SnapshotID != "snap-1" { + t.Fatalf("record %s, want the first endpoint's checkpoint snap-1", rec.SnapshotID) + } + if got, want := second.steps(), "record,list,ready"; got != want { + t.Fatalf("steps after the restart %s, want %s", got, want) + } +} + +// TestRestoreThroughRestate: a host restore that fails once is retried, and a +// host that refuses for good marks the row error and fails the restore. +func TestRestoreThroughRestate(t *testing.T) { + fleet := newSnapFleet() + fleet.fail["restore"] = []error{errors.New("agent 503")} + client := serveFleet(t, fleet, fastPolicy) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := client.Restore(ctx, uniqueVM("restore"), "snap-1"); err != nil { + t.Fatalf("restore: %v", err) + } + if got, want := fleet.steps(), "validate,restoring,restore,restore,finish"; got != want { + t.Fatalf("steps %s, want %s", got, want) + } + + refused := newSnapFleet() + refused.fail["restore"] = []error{&orchestrator.HTTPStatusError{Code: 409, Body: "vm busy"}} + client = serveFleet(t, refused, fastPolicy) + if err := client.Restore(ctx, uniqueVM("restore-refused"), "snap-1"); err == nil || !strings.Contains(err.Error(), "vm busy") { + t.Fatalf("restore error %v, want the agent's refusal", err) + } + if got, want := refused.steps(), "validate,restoring,restore,failed"; got != want { + t.Fatalf("steps %s, want %s", got, want) + } + if !strings.Contains(refused.failed, "vm busy") { + t.Fatalf("row error %q, want the refusal", refused.failed) + } +} + +// TestRestoreSurvivesEndpointRestart stands in for kill -9 during a +// restore: the restarted endpoint runs the restore to the end instead of +// leaving the row in restoring. +func TestRestoreSurvivesEndpointRestart(t *testing.T) { + first := newSnapFleet() + first.block["restore"] = make(chan struct{}) + defer close(first.block["restore"]) + client, addr, stop := serveFleetAt(t, first, "") + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- client.Restore(ctx, uniqueVM("restore-restart"), "snap-1") }() + <-first.entered + stop() + + second := newSnapFleet() + _, _, _ = serveFleetAt(t, second, addr) + if err := <-done; err != nil { + t.Fatalf("restore: %v", err) + } + if got, want := second.steps(), "restore,finish"; got != want { + t.Fatalf("steps after the restart %s, want %s", got, want) + } +} + +// TestCollectSnapshotRecordsFailureWhenRetriesRunOut: a gc deletion that +// keeps failing is retried under the policy, then the row is marked error. +func TestCollectSnapshotRecordsFailureWhenRetriesRunOut(t *testing.T) { + fleet := newSnapFleet() + fleet.fail["collect"] = []error{errAlways} + client := serveFleet(t, fleet, fastPolicy) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + key := fmt.Sprintf("collect:snap-1:%d", time.Now().UnixNano()) + if err := client.SendCollectSnapshot(ctx, uniqueVM("gc"), "snap-1", key); err != nil { + t.Fatal(err) + } + want := strings.TrimSuffix(strings.Repeat("collect,", fastPolicy.MaxAttempts), ",") + ",collect_failed" + deadline := time.Now().Add(10 * time.Second) + for fleet.steps() != want { + if time.Now().After(deadline) { + t.Fatalf("steps %s, want %s", fleet.steps(), want) + } + time.Sleep(50 * time.Millisecond) + } + if !strings.Contains(fleet.failed, "store down") { + t.Fatalf("row error %q, want the last delete error", fleet.failed) + } +} + +// TestCheckpointChainThroughRestate: a tick on an empty chain takes a full +// checkpoint and sends its copy to Artifacts, which reports back to +// ChainMoved; the copy becomes the base, pinned on restate. the next tick +// merges a diff onto it, moving the pin, and a destroy retires the chain. +func TestCheckpointChainThroughRestate(t *testing.T) { + fleet := newSnapFleet() + client := serveFleet(t, fleet, fastPolicy) + vm := uniqueVM("chain") + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := client.Checkpoint(ctx, vm); err != nil { + t.Fatalf("first tick: %v", err) + } + chain := fleet.waitSettled(t) + if chain.Standby != "h2" || chain.Head.SnapshotID != "snap-1" || chain.Base == "" { + t.Fatalf("chain after the first tick %+v, want snap-1 copied to h2", chain) + } + firstBase := chain.Base + if p := pinState(t, ctx, client, []string{firstBase}); !p[0] { + t.Fatalf("base %s not pinned", firstBase) + } + + if err := client.Checkpoint(ctx, vm); err != nil { + t.Fatalf("second tick: %v", err) + } + chain = fleet.waitSettled(t) + if chain.Head.SnapshotID != "snap-2" || chain.Base == firstBase || chain.Base == "" { + t.Fatalf("chain after the diff %+v, want snap-2 merged onto a new base", chain) + } + waitUnpinned(t, ctx, client, []string{firstBase}) + if p := pinState(t, ctx, client, []string{chain.Base}); !p[0] { + t.Fatalf("new base %s not pinned", chain.Base) + } + want := "plan,begin,checkpoint,record,list,ready,plan_move,advanced," + + "plan,begin,checkpoint,record,list,ready,plan_move,retire,advanced" + if got := fleet.steps(); got != want { + t.Fatalf("steps %s, want %s", got, want) + } + + if err := client.Destroy(ctx, vm, orchestrator.DestroyRequest{Reason: "manual_destroy"}); err != nil { + t.Fatalf("destroy: %v", err) + } + waitUnpinned(t, ctx, client, []string{chain.Base}) + fleet.mu.Lock() + last := fleet.retired[len(fleet.retired)-1] + fleet.mu.Unlock() + if last.Base != chain.Base || last.Head.SnapshotID != "snap-2" { + t.Fatalf("destroy retired %+v, want the whole chain", last) + } + if c, _ := fleet.mirrored(); !c.Empty() { + t.Fatalf("mirror %+v after the destroy, want empty", c) + } +} + +// TestHoldChainKeepsTheTickOff: while a migrate holds the chain a tick does +// nothing, and once released it runs again. +func TestHoldChainKeepsTheTickOff(t *testing.T) { + fleet := newSnapFleet() + client := serveFleet(t, fleet, fastPolicy) + vm := uniqueVM("hold") + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + hold, err := client.HoldChain(ctx, vm, time.Hour) + if err != nil || hold.Busy { + t.Fatalf("hold = %+v, %v", hold, err) + } + if err := client.Checkpoint(ctx, vm); err != nil { + t.Fatal(err) + } + if got := fleet.steps(); got != "" { + t.Fatalf("held tick ran steps %s", got) + } + if err := client.ReleaseChain(ctx, vm, false); err != nil { + t.Fatal(err) + } + if err := client.Checkpoint(ctx, vm); err != nil { + t.Fatal(err) + } + fleet.waitSettled(t) + if got := fleet.steps(); !strings.HasPrefix(got, "plan,") { + t.Fatalf("released tick ran %q, want it to plan", got) + } +} + +// TestCheckpointChainFailedCopyRetiresADiff: a delta merge that fails for +// good deletes the child and retires the chain, whose base it may have +// consumed; the pin on that base goes with it. +func TestCheckpointChainFailedCopyRetiresADiff(t *testing.T) { + fleet := newSnapFleet() + client := serveFleet(t, fleet, fastPolicy) + vm := uniqueVM("chain-fail") + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := client.Checkpoint(ctx, vm); err != nil { + t.Fatal(err) + } + base := fleet.waitSettled(t).Base + + failing := &failingPullFleet{snapFleet: fleet} + client = serveFleet(t, failing, fastPolicy) + if err := client.Checkpoint(ctx, vm); err != nil { + t.Fatal(err) + } + if chain := fleet.waitSettled(t); !chain.Empty() { + t.Fatalf("chain %+v after a failed merge, want it retired", chain) + } + waitUnpinned(t, ctx, client, []string{base}) + fleet.mu.Lock() + retired := fleet.retired + fleet.mu.Unlock() + if len(retired) != 2 || retired[0].Head.SnapshotID != "snap-2" || retired[1].Base != base { + t.Fatalf("retired %+v, want the child then the chain on %s", retired, base) + } +} + +// failingPullFleet refuses every pull for good. +type failingPullFleet struct{ *snapFleet } + +func (f *failingPullFleet) ArtifactPull(context.Context, orchestrator.ArtifactMoveJob) (orchestrator.ArtifactMoved, error) { + return orchestrator.ArtifactMoved{}, &orchestrator.HTTPStatusError{Code: 409, Body: "digest mismatch"} +} diff --git a/internal/orchestrator/checkpoints_durable_test.go b/internal/orchestrator/checkpoints_durable_test.go new file mode 100644 index 0000000..3ab9555 --- /dev/null +++ b/internal/orchestrator/checkpoints_durable_test.go @@ -0,0 +1,145 @@ +package orchestrator + +import ( + "context" + "strings" + "testing" +) + +// TestChainSteps_fullThenDiff drives the durable chain's steps by hand the +// way the Environment handler does: an empty chain plans a full checkpoint +// to the standby, the recorded copy becomes the base, and the next tick plans +// a diff against the head. +func TestChainSteps_fullThenDiff(t *testing.T) { + provider := newDeltaMigrateProvider() + fm := newDeltaFleet(t, provider, &deltaMover{}) + ctx := context.Background() + vmID := "fuse-task-src" + + plan, err := fm.ChainBegin(ctx, vmID, ChainState{}) + if err != nil || plan.Diff || plan.Gone || plan.Standby != "h-b" { + t.Fatalf("first plan = %+v, %v; want a full checkpoint to h-b", plan, err) + } + full, err := fm.snapshotInProcess(ctx, vmID, plan.Options) + if err != nil { + t.Fatal(err) + } + mv, err := fm.ChainMovePlan(ctx, ChainState{}, plan, full) + if err != nil || mv.Existing != "" || mv.Job.HostID != "h-b" || mv.Job.Base != "" { + t.Fatalf("full move = %+v, %v; want a whole copy to h-b", mv, err) + } + moved, err := fm.ArtifactPull(ctx, mv.Job) + if err != nil { + t.Fatal(err) + } + base, err := fm.ArtifactRecord(ctx, mv.Job, moved) + if err != nil { + t.Fatal(err) + } + chain := ChainState{Standby: "h-b", Head: full, Base: base, Bytes: full.SizeBytes} + fm.MirrorChain(vmID, chain, false) + if got := fm.standbyCopies()[vmID]; got.host != "h-b" { + t.Fatalf("mirrored standby = %+v, want h-b", got) + } + + plan, err = fm.ChainBegin(ctx, vmID, chain) + if err != nil || !plan.Diff || plan.Options.Parent != full.SnapshotID || plan.Standby != "h-b" { + t.Fatalf("second plan = %+v, %v; want a diff against %s", plan, err, full.SnapshotID) + } + child, err := fm.snapshotInProcess(ctx, vmID, plan.Options) + if err != nil { + t.Fatal(err) + } + mv, err = fm.ChainMovePlan(ctx, chain, plan, child) + if err != nil || mv.Job.Base != base || !mv.Job.DropBase { + t.Fatalf("delta move = %+v, %v; want a merge onto %s", mv.Job, err, base) + } + + // retiring the chain removes the base's record and the head. + if err := fm.ChainRetire(ctx, vmID, chain); err != nil { + t.Fatal(err) + } + if _, err := fm.GetSnapshotByID(ctx, base); err == nil { + t.Errorf("retired base %s still recorded", base) + } + if _, err := fm.GetSnapshotByID(ctx, full.SnapshotID); err == nil { + t.Errorf("retired head %s still recorded", full.SnapshotID) + } +} + +// TestChainBegin_goneVM: a vm the fleet dropped plans its chain's removal. +func TestChainBegin_goneVM(t *testing.T) { + fm := newDeltaFleet(t, newDeltaMigrateProvider(), &deltaMover{}) + plan, err := fm.ChainBegin(context.Background(), "fuse-nope", ChainState{Base: "b"}) + if err != nil || !plan.Gone { + t.Fatalf("plan = %+v, %v; want gone", plan, err) + } +} + +// TestCheckpointTickDurable: each candidate's tick goes to its object, and a +// mirrored chain whose vm is gone is sent to be retired. +func TestCheckpointTickDurable(t *testing.T) { + fm := newDeltaFleet(t, newDeltaMigrateProvider(), &deltaMover{}) + d := &fakeDurableSnapshots{} + fm.MirrorChain("fuse-gone", ChainState{Standby: "h-b", Base: "b"}, false) + + fm.checkpointTickDurable(context.Background(), d) + if got := strings.Join(d.checkpoints, ","); got != "fuse-task-src" { + t.Fatalf("checkpoint calls %q, want fuse-task-src", got) + } + if got := strings.Join(d.retired, ","); got != "fuse-gone" { + t.Fatalf("retire sends %q, want fuse-gone", got) + } + if _, ok := fm.standbyCopies()["fuse-gone"]; ok { + t.Fatal("gone chain still mirrored") + } +} + +// TestMigrateVM_durableHoldsTheChain: a delta migrate on restate takes the +// chain from the vm's object, moves only the diff onto the staged base, and +// releases the hold with a reset since the merge consumed the base. +func TestMigrateVM_durableHoldsTheChain(t *testing.T) { + provider := newDeltaMigrateProvider() + mover := &deltaMover{} + fm := newDeltaFleet(t, provider, mover) + ctx := context.Background() + + // build a real chain in process, then hand it out through the hold. + fm.advanceChain(ctx, "fuse-task-src") + c := fm.chainFor("fuse-task-src") + held := ChainState{Standby: c.standby, Head: c.head, Base: c.base, Bytes: c.bytes} + d := &fakeDurableSnapshots{hold: ChainHold{Chain: held}} + fm.SetDurableSnapshots(d) + + if _, err := fm.MigrateVM(ctx, "fuse-task-src", MigrateOptions{TargetHostID: "h-b", Live: true}); err != nil { + t.Fatalf("migrate: %v", err) + } + moves := mover.calls() + if len(moves) != 2 || moves[1].Base != held.Base { + t.Fatalf("moves = %+v, want the chain's full copy then one delta onto %s", moves, held.Base) + } + if len(d.holds) != 1 || len(d.releases) != 1 || !d.releases[0] { + t.Fatalf("holds %v releases %v, want one hold released with a reset", d.holds, d.releases) + } +} + +// TestMigrateVM_durableBusyChainTakesAFullSeed: a chain whose copy is still +// in flight is not held, and the migrate seeds from a fresh full snapshot. +func TestMigrateVM_durableBusyChainTakesAFullSeed(t *testing.T) { + provider := newDeltaMigrateProvider() + mover := &deltaMover{} + fm := newDeltaFleet(t, provider, mover) + d := &fakeDurableSnapshots{hold: ChainHold{Busy: true}} + fm.SetDurableSnapshots(d) + + if _, err := fm.MigrateVM(context.Background(), "fuse-task-src", MigrateOptions{TargetHostID: "h-b", Live: true}); err != nil { + t.Fatalf("migrate: %v", err) + } + moves := mover.calls() + if len(moves) != 1 || moves[0].Base != "" { + t.Fatalf("moves = %+v, want one full seed copy", moves) + } + if len(d.releases) != 0 { + t.Fatalf("released a hold that was never taken: %v", d.releases) + } +} diff --git a/internal/orchestrator/snapshot_durable_test.go b/internal/orchestrator/snapshot_durable_test.go new file mode 100644 index 0000000..f6697c4 --- /dev/null +++ b/internal/orchestrator/snapshot_durable_test.go @@ -0,0 +1,295 @@ +package orchestrator + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" +) + +// fakeDurableSnapshots stands in for the restate client: it runs nothing and +// records what it was asked to do. +type fakeDurableSnapshots struct { + mu sync.Mutex + snapshots []string + restores []string + checkpoints []string + retired []string + collects []string + holds []string + releases []bool + hold ChainHold + holdErr error +} + +func (f *fakeDurableSnapshots) Snapshot(_ context.Context, vmID string, _ SnapshotOptions) (SnapshotRecord, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.snapshots = append(f.snapshots, vmID) + return SnapshotRecord{SnapshotID: "snap-durable", VMID: vmID, State: SnapshotStateReady}, nil +} + +func (f *fakeDurableSnapshots) Restore(_ context.Context, vmID, snapshotID string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.restores = append(f.restores, vmID+"/"+snapshotID) + return nil +} + +func (f *fakeDurableSnapshots) Checkpoint(_ context.Context, vmID string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.checkpoints = append(f.checkpoints, vmID) + return nil +} + +func (f *fakeDurableSnapshots) HoldChain(_ context.Context, vmID string, _ time.Duration) (ChainHold, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.holds = append(f.holds, vmID) + return f.hold, f.holdErr +} + +func (f *fakeDurableSnapshots) ReleaseChain(_ context.Context, _ string, reset bool) error { + f.mu.Lock() + defer f.mu.Unlock() + f.releases = append(f.releases, reset) + return nil +} + +func (f *fakeDurableSnapshots) SendRetireChain(_ context.Context, vmID string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.retired = append(f.retired, vmID) + return nil +} + +func (f *fakeDurableSnapshots) SendCollectSnapshot(_ context.Context, vmID, snapshotID, key string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.collects = append(f.collects, vmID+"/"+snapshotID+"/"+key) + return nil +} + +// TestSnapshotAndRestore_durable: with restate configured the api's snapshot +// and restore run on the Environment object, and a fork's seed snapshot stays +// in process (fork is not on restate yet, and a handler on the same object +// must never call it). +func TestSnapshotAndRestore_durable(t *testing.T) { + ctx := context.Background() + provider := newSnapshotTestProvider() + fm := NewFleetManager(FleetConfig{Provider: provider, Prefix: "fuse-"}) + vmID := provisionSnapshotTestVM(t, fm, "task-durable") + d := &fakeDurableSnapshots{} + fm.SetDurableSnapshots(d) + + rec, err := fm.CreateSnapshot(ctx, vmID, SnapshotOptions{Comment: "x"}) + if err != nil || rec.SnapshotID != "snap-durable" { + t.Fatalf("create = %+v, %v; want the durable record", rec, err) + } + if err := fm.RestoreSnapshot(ctx, vmID, "snap-durable"); err != nil { + t.Fatalf("restore: %v", err) + } + if got := strings.Join(d.snapshots, ","); got != vmID { + t.Fatalf("durable snapshots %q, want %q", got, vmID) + } + if got := strings.Join(d.restores, ","); got != vmID+"/snap-durable" { + t.Fatalf("durable restores %q", got) + } + if n := len(provider.envs[vmID].checkpoints); n != 0 { + t.Fatalf("%d in-process checkpoints, want 0", n) + } + + if _, err := fm.snapshotInProcess(ctx, vmID, SnapshotOptions{}); err != nil { + t.Fatalf("in-process snapshot: %v", err) + } + if n := len(provider.envs[vmID].checkpoints); n != 1 { + t.Fatalf("%d in-process checkpoints, want 1", n) + } + if len(d.snapshots) != 1 { + t.Fatalf("in-process snapshot went to restate") + } +} + +// TestSnapshotSteps_abortDeletesArtifactAndMarksError: the compensation of a +// snapshot whose record could not be finished leaves nothing on the host and +// the row in error with the cause. +func TestSnapshotSteps_abortDeletesArtifactAndMarksError(t *testing.T) { + ctx := context.Background() + provider := newSnapshotTestProvider() + fm := NewFleetManager(FleetConfig{Provider: provider, Prefix: "fuse-"}) + vmID := provisionSnapshotTestVM(t, fm, "task-abort") + + plan, err := fm.SnapshotBegin(ctx, vmID, SnapshotOptions{Comment: "c"}) + if err != nil { + t.Fatalf("begin: %v", err) + } + created, err := fm.SnapshotCheckpoint(ctx, vmID, plan) + if err != nil { + t.Fatalf("checkpoint: %v", err) + } + if _, err := fm.SnapshotRecordCreating(ctx, vmID, plan, created); err != nil { + t.Fatalf("record: %v", err) + } + if err := fm.SnapshotAbort(ctx, vmID, plan, created, "ready write failed"); err != nil { + t.Fatalf("abort: %v", err) + } + if n := len(provider.envs[vmID].checkpoints); n != 0 { + t.Fatalf("%d checkpoints left on the host, want 0", n) + } + rec, err := fm.GetSnapshotByID(ctx, created.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if rec.State != SnapshotStateError || rec.LastError != "ready write failed" { + t.Fatalf("row %s %q, want error with the cause", rec.State, rec.LastError) + } +} + +// TestPermanentSnapshotError: refusals a retry cannot change are terminal, +// host failures and timeouts are retried. +func TestPermanentSnapshotError(t *testing.T) { + for _, tc := range []struct { + err error + want bool + }{ + {fmt.Errorf("%w: fuse-x", ErrVMNotRunning), true}, + {fmt.Errorf("%w: q", ErrSnapshotQuotaExceeded), true}, + {fmt.Errorf("%w: vm", errSnapshotOptions), true}, + {fmt.Errorf("snapshot: %w", &HTTPStatusError{Code: 409, Body: "not the chain head"}), true}, + {&HTTPStatusError{Code: 408}, false}, + {&HTTPStatusError{Code: 503}, false}, + {errors.New("connection reset"), false}, + {nil, false}, + } { + if got := PermanentSnapshotError(tc.err); got != tc.want { + t.Errorf("PermanentSnapshotError(%v) = %v, want %v", tc.err, got, tc.want) + } + } +} + +// TestRestoreSteps_validateRejectsNonReady: a restore validates the row +// before anything is marked. +func TestRestoreSteps_validateRejectsNonReady(t *testing.T) { + ctx := context.Background() + provider := newSnapshotTestProvider() + fm := NewFleetManager(FleetConfig{Provider: provider, Prefix: "fuse-"}) + vmID := provisionSnapshotTestVM(t, fm, "task-restore") + rec, err := fm.CreateSnapshot(ctx, vmID, SnapshotOptions{}) + if err != nil { + t.Fatal(err) + } + got, err := fm.RestoreValidate(ctx, vmID, rec.SnapshotID) + if err != nil || got.SnapshotID != rec.SnapshotID { + t.Fatalf("validate = %+v, %v", got, err) + } + if err := fm.RestoreBegin(ctx, got); err != nil { + t.Fatal(err) + } + if _, err := fm.RestoreValidate(ctx, vmID, rec.SnapshotID); !errors.Is(err, ErrSnapshotInvalidState) { + t.Fatalf("validate of a restoring row: %v, want ErrSnapshotInvalidState", err) + } + if err := fm.RestoreOnHost(ctx, vmID, rec.SnapshotID); err != nil { + t.Fatalf("restore on host: %v", err) + } + if err := fm.RestoreFinish(ctx, got); err != nil { + t.Fatal(err) + } + after, _ := fm.GetSnapshotByID(ctx, rec.SnapshotID) + if after.State != SnapshotStateReady { + t.Fatalf("row %s after the restore, want ready", after.State) + } +} + +// TestReconcileSnapshots_durableSendsCollect: on restate an expired leaf is +// marked deleting and handed to the Environment object, keyed on the +// marking, instead of being deleted in process. +func TestReconcileSnapshots_durableSendsCollect(t *testing.T) { + ctx := context.Background() + provider := newSnapshotTestProvider() + fm := NewFleetManager(FleetConfig{Provider: provider, Prefix: "fuse-"}) + vmID := provisionSnapshotTestVM(t, fm, "task-gc") + first, _ := fm.CreateSnapshot(ctx, vmID, SnapshotOptions{}) + // no lineage, or first would be kept as the second's parent. + if _, err := fm.CreateSnapshot(ctx, vmID, SnapshotOptions{NoLineage: true}); err != nil { + t.Fatal(err) + } + expired := time.Now().Add(-time.Hour) + first.RetentionUntil = &expired + if err := fm.upsertSnapshotRecord(ctx, first); err != nil { + t.Fatal(err) + } + d := &fakeDurableSnapshots{} + fm.SetDurableSnapshots(d) + + fm.reconcileSnapshots(ctx) + fm.reconcileSnapshots(ctx) + if len(d.collects) != 2 || d.collects[0] != d.collects[1] { + t.Fatalf("collect sends %v, want two with the same key", d.collects) + } + if !strings.HasPrefix(d.collects[0], vmID+"/"+first.SnapshotID+"/collect:"+first.SnapshotID+":") { + t.Fatalf("collect send %q", d.collects[0]) + } + rec, _ := fm.GetSnapshotByID(ctx, first.SnapshotID) + if rec.State != SnapshotStateDeleting { + t.Fatalf("row %s, want deleting until the object collects it", rec.State) + } + if n := len(provider.envs[vmID].checkpoints); n != 2 { + t.Fatalf("%d checkpoints, want both still there", n) + } + + if err := fm.SnapshotCollect(ctx, first.SnapshotID); err != nil { + t.Fatalf("collect: %v", err) + } + if _, err := fm.GetSnapshotByID(ctx, first.SnapshotID); err == nil { + t.Fatal("collected row still there") + } + // a repeat after the row is gone does nothing. + if err := fm.SnapshotCollect(ctx, first.SnapshotID); err != nil { + t.Fatalf("collect again: %v", err) + } +} + +// TestReconcileSnapshots_skipsPinned: retention gc leaves a checkpoint that a +// move is still reading, in process or on restate. +func TestReconcileSnapshots_skipsPinned(t *testing.T) { + ctx := context.Background() + provider := newSnapshotTestProvider() + fm := NewFleetManager(FleetConfig{Provider: provider, Prefix: "fuse-"}) + vmID := provisionSnapshotTestVM(t, fm, "task-pin") + first, _ := fm.CreateSnapshot(ctx, vmID, SnapshotOptions{}) + // no lineage, or first would be kept as the second's parent. + if _, err := fm.CreateSnapshot(ctx, vmID, SnapshotOptions{NoLineage: true}); err != nil { + t.Fatal(err) + } + expired := time.Now().Add(-time.Hour) + first.RetentionUntil = &expired + if err := fm.upsertSnapshotRecord(ctx, first); err != nil { + t.Fatal(err) + } + + fm.beginArtifactPull(first.SnapshotID) + fm.reconcileSnapshots(ctx) + if _, err := fm.GetSnapshotByID(ctx, first.SnapshotID); err != nil { + t.Fatalf("in-process pinned checkpoint collected: %v", err) + } + fm.endArtifactPull(first.SnapshotID) + + da := newFakeDurableArtifacts() + fm.SetDurableArtifacts(da) + if err := da.PinArtifacts(ctx, "move:x", 0, first.SnapshotID); err != nil { + t.Fatal(err) + } + fm.reconcileSnapshots(ctx) + if _, err := fm.GetSnapshotByID(ctx, first.SnapshotID); err != nil { + t.Fatalf("durably pinned checkpoint collected: %v", err) + } + _ = da.ReleaseArtifacts(ctx, "move:x", first.SnapshotID) + fm.reconcileSnapshots(ctx) + if _, err := fm.GetSnapshotByID(ctx, first.SnapshotID); err == nil { + t.Fatal("released checkpoint not collected") + } +} From 50cef637f768d0b6f1c57564382e067292f0636f Mon Sep 17 00:00:00 2001 From: andrewn6 <59238070+andrewn6@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:33:41 -0500 Subject: [PATCH 4/6] docs: snapshot, restore and checkpoint chain on restate --- .../docs/cloud/orchestrator-deploy.mdx | 4 +-- docs/content/docs/concepts/snapshots.mdx | 11 ++++++++ docs/runbooks/restate.mdx | 27 +++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/content/docs/cloud/orchestrator-deploy.mdx b/docs/content/docs/cloud/orchestrator-deploy.mdx index 05657a5..7c375f9 100644 --- a/docs/content/docs/cloud/orchestrator-deploy.mdx +++ b/docs/content/docs/cloud/orchestrator-deploy.mdx @@ -71,7 +71,7 @@ range covers the fleet. | `FUSE_RESTATE_INGRESS_URL` | | | Restate ingress URL, required unless the mode is `off`. `http://127.0.0.1:18080` with `ops/restate/compose.yaml`. | | `FUSE_RESTATE_API_KEY` / `FUSE_RESTATE_IDENTITY_KEY` | | | Mode `cloud` only: the Cloud API key and the `publickeyv1_...` signing key. Other cloud settings are on [Restate Cloud](/docs/cloud/restate-cloud#settings). | | `ORCH_RESTATE_LISTEN` | | `127.0.0.1:9080` | Restate handler endpoint (h2c). Only restate-server, or the TLS proxy in front of it in cloud public mode, should reach it. | -| `FUSE_RESTATE_RETRY__INITIAL_INTERVAL` / `_MAX_INTERVAL` / `_MAX_ATTEMPTS` | | `500ms` / `30s` / `10` | Invocation retry policy per restate service; `` is `ENVIRONMENT`, `HOST`, `ARTIFACTS` or `ARTIFACTPIN`. The `ARTIFACTS` policy also bounds the retries of an artifact move's pull. | +| `FUSE_RESTATE_RETRY__INITIAL_INTERVAL` / `_MAX_INTERVAL` / `_MAX_ATTEMPTS` | | `500ms` / `30s` / `10` | Invocation retry policy per restate service; `` is `ENVIRONMENT`, `HOST`, `ARTIFACTS` or `ARTIFACTPIN`. The `ARTIFACTS` policy also bounds the retries of an artifact move's pull; the `ENVIRONMENT` policy bounds the host steps of a destroy, snapshot or restore, and the retries of a snapshot gc deletion. | With `FIRECRACKER_BASE_URL` unset, the orchestrator runs the in-memory stub provider (fine for local dev, not for production). With `DATABASE_URL` unset, state is @@ -79,7 +79,7 @@ in-memory and does not survive a restart. ## Restate -Restate is optional and off by default; nothing runs on it yet. To run it next to the orchestrator: +Restate is optional and off by default. With it on, destroys, artifact moves, snapshots, restores, the checkpoint chain and snapshot gc run on it. To run it next to the orchestrator: ```bash docker compose -f ops/restate/compose.yaml up -d diff --git a/docs/content/docs/concepts/snapshots.mdx b/docs/content/docs/concepts/snapshots.mdx index 7cf7a32..fc0d3e5 100644 --- a/docs/content/docs/concepts/snapshots.mdx +++ b/docs/content/docs/concepts/snapshots.mdx @@ -101,6 +101,17 @@ Restore is **in-place only** for both kinds: it restores the snapshot onto its original VM, which must still be running. There is no cross-host restore, clone, or standalone restore into a new VM. +### With restate on + +With `FUSE_RESTATE_MODE=self` or `cloud`, a snapshot and a restore each run as +one durable operation on the environment, one at a time per environment. If the +orchestrator restarts mid-way, the operation picks up where it stopped: a +snapshot whose host checkpoint was taken still gets its record, and a restore +finishes instead of leaving the snapshot in `restoring`. A snapshot whose record +cannot be written is undone: the host artifact is deleted and the snapshot is +left in `error` with the cause in `last_error`. The API, its fields and its +status codes are the same in both modes. + ### The guest clock after a live restore A resumed guest's clock is frozen at the moment the snapshot was taken, and diff --git a/docs/runbooks/restate.mdx b/docs/runbooks/restate.mdx index f4e0ef3..e1f0138 100644 --- a/docs/runbooks/restate.mdx +++ b/docs/runbooks/restate.mdx @@ -158,6 +158,33 @@ forgets its chain instead of deleting it. q "SELECT service_key, key, value_utf8 FROM state WHERE service_name = 'ArtifactPin'" ``` +snapshots and restores run as `Environment//Snapshot` and +`Environment//Restore`. the host checkpoint carries the host-agent +idempotency key, so a restart between it and the row write records the +checkpoint already taken. a snapshot whose record cannot be finished deletes +the host artifact and leaves the row `error` (`snapshot.failed` event). a +restore resumes at its first unfinished step after a restart. + +the checkpoint loop runs each vm's tick as `Environment//Checkpoint`. +the chain (head, base, standby) is that object's state, so a restart keeps it, +and its base is pinned in `ArtifactPin` with holder `chain:`. the copy +to the standby is a one-way `Artifacts` move that reports back to +`Environment//ChainMoved`, so a long copy does not hold the +environment's other ops. a delta migrate takes `HoldChain` (a 1h lease) for +the whole migrate; a tick on a held chain does nothing. a destroy drops the +chain. a copy that never reports back (a killed `Artifacts` invocation) is +forgotten after 2h without deleting anything. + +```sh +q "SELECT service_key, value_utf8 FROM state WHERE service_name = 'Environment' AND key = 'chain'" +``` + +retention gc marks an expired snapshot `deleting` and sends +`Environment//CollectSnapshot`, which retries the host delete under the +`ENVIRONMENT` policy and only then moves the row to `error` +(`snapshot.gc_failed`); the next sweep marks it again. gc skips a snapshot any +pin holds, in process or in `ArtifactPin`. + copies are metered by `orchestrator_artifact_move_duration_seconds`, `orchestrator_artifact_move_bytes_total` and `orchestrator_artifact_move_failures_total`, labeled `source_host` and From 07dd6c5a3094a41b6ba8eb95e9e57f086ff3f49a Mon Sep 17 00:00:00 2001 From: andrewn6 <59238070+andrewn6@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:40:20 -0500 Subject: [PATCH 5/6] fix: make chain holds exclusive and owned, retry a dropped chain's cleanup, and re-check pins before snapshot gc deletes --- docs/runbooks/restate.mdx | 3 +- internal/durable/checkpoint.go | 52 +++++++++++++++---- internal/durable/server.go | 4 +- internal/durable/snapshot_test.go | 16 +++++- internal/orchestrator/checkpoints_durable.go | 25 +++++---- internal/orchestrator/migrate.go | 5 +- internal/orchestrator/snapshot_durable.go | 23 ++++++-- .../orchestrator/snapshot_durable_test.go | 20 ++++++- 8 files changed, 116 insertions(+), 32 deletions(-) diff --git a/docs/runbooks/restate.mdx b/docs/runbooks/restate.mdx index e1f0138..872ef56 100644 --- a/docs/runbooks/restate.mdx +++ b/docs/runbooks/restate.mdx @@ -171,7 +171,8 @@ and its base is pinned in `ArtifactPin` with holder `chain:`. the copy to the standby is a one-way `Artifacts` move that reports back to `Environment//ChainMoved`, so a long copy does not hold the environment's other ops. a delta migrate takes `HoldChain` (a 1h lease) for -the whole migrate; a tick on a held chain does nothing. a destroy drops the +the whole migrate; a tick on a held chain does nothing, and a second migrate +finds it busy and takes a full seed. a destroy drops the chain. a copy that never reports back (a killed `Artifacts` invocation) is forgotten after 2h without deleting anything. diff --git a/internal/durable/checkpoint.go b/internal/durable/checkpoint.go index 3a12cb7..59a8184 100644 --- a/internal/durable/checkpoint.go +++ b/internal/durable/checkpoint.go @@ -38,6 +38,8 @@ type chainObject struct { // HeldUntil is a migrate's hold, unix ms. ticks skip the chain until it // passes or the migrate releases it. HeldUntil int64 `json:"held_until,omitempty"` + // HeldBy is the hold's id, which its release has to name. + HeldBy string `json:"held_by,omitempty"` // Dropped is a vm destroyed while a copy or a hold was in flight: what // the chain holds is removed when that ends. Dropped bool `json:"dropped,omitempty"` @@ -144,7 +146,7 @@ func checkpointTick(ctx restate.ObjectContext, fleet Fleet, policy RetryPolicy) // a migrate of this vm is in progress. return nil } - st.HeldUntil = 0 + st.HeldUntil, st.HeldBy = 0, "" plan, err := Run(ctx, opCheckpoint, "plan", func(c context.Context) (orchestrator.ChainPlan, error) { return fleet.ChainBegin(c, vmID, st.Chain) @@ -344,11 +346,27 @@ func dropChain(ctx restate.ObjectContext, fleet Checkpoints) error { if st.Pending != nil || st.HeldUntil > now { st.Dropped = true restate.Set(ctx, chainKey, st) + // the copy's report or the hold's release normally ends it. this + // delayed retry comes back when the hold or the copy's lapse is due, + // so a migrate or a move that died with its process does not leave + // the chain dropped forever. + restate.ObjectSend(ctx, EnvironmentObject, vmID, "RetireChain"). + Send(restate.Void{}, restate.WithDelay(dropRetryDelay(st, now))) return nil } return dropChainNow(ctx, fleet, vmID, st) } +// dropRetryDelay is how long until a dropped chain's hold lapses or its copy +// counts as never reporting back, whichever comes later, plus a margin. +func dropRetryDelay(st chainObject, now int64) time.Duration { + due := st.HeldUntil + if st.Pending != nil { + due = max(due, st.Pending.Since+pendingLapse.Milliseconds()) + } + return time.Duration(max(due-now, 0))*time.Millisecond + time.Minute +} + // holdChain is Environment/{vm_id}.HoldChain, taken by a delta migrate. func holdChain(ctx restate.ObjectContext, fleet Checkpoints, lease time.Duration) (orchestrator.ChainHold, error) { st, err := loadChain(ctx) @@ -362,21 +380,30 @@ func holdChain(ctx restate.ObjectContext, fleet Checkpoints, lease time.Duration if err != nil { return orchestrator.ChainHold{}, err } - st.HeldUntil = now + lease.Milliseconds() + if st.HeldUntil > now { + // another migrate holds it: one chain, one consumer of its base. + return orchestrator.ChainHold{Busy: true}, nil + } + st.HeldUntil, st.HeldBy = now+lease.Milliseconds(), ctx.Request().ID saveChain(ctx, fleet, st) - return orchestrator.ChainHold{Chain: st.Chain}, nil + return orchestrator.ChainHold{Holder: st.HeldBy, Chain: st.Chain}, nil } // releaseChain is Environment/{vm_id}.ReleaseChain. reset forgets the chain // without deleting its base, which the migrate consumed or will leave to gc. -func releaseChain(ctx restate.ObjectContext, fleet Checkpoints, reset bool) error { +// a release that names any other hold than the current one (one that lapsed +// and was taken again) changes nothing. +func releaseChain(ctx restate.ObjectContext, fleet Checkpoints, req ReleaseRequest) error { vmID := restate.Key(ctx) st, err := loadChain(ctx) if err != nil { return err } - st.HeldUntil = 0 - if reset { + if req.Holder == "" || req.Holder != st.HeldBy { + return nil + } + st.HeldUntil, st.HeldBy = 0, "" + if req.Reset { if st.Chain.Base != "" { restate.ObjectSend(ctx, ArtifactPinObject, st.Chain.Base, "Release").Send(chainHolder(vmID)) } @@ -394,6 +421,13 @@ type HoldRequest struct { Lease time.Duration `json:"lease"` } +// ReleaseRequest is the input of ReleaseChain: the hold it ends, and whether +// the chain is forgotten with it. +type ReleaseRequest struct { + Holder string `json:"holder"` + Reset bool `json:"reset,omitempty"` +} + // Checkpoint runs Environment/{vmID}.Checkpoint and waits for it. func (c *Client) Checkpoint(ctx context.Context, vmID string) error { _, err := ingress.Object[restate.Void, restate.Void](c.longIngress, EnvironmentObject, vmID, "Checkpoint"). @@ -412,9 +446,9 @@ func (c *Client) HoldChain(ctx context.Context, vmID string, lease time.Duration // ReleaseChain enqueues Environment/{vmID}.ReleaseChain without waiting: it // may queue behind a long op on the vm, and a hold that is not released only // lapses on its lease. -func (c *Client) ReleaseChain(ctx context.Context, vmID string, reset bool) error { - _, err := ingress.ObjectSend[bool](c.ingress, EnvironmentObject, vmID, "ReleaseChain"). - Send(ctx, reset) +func (c *Client) ReleaseChain(ctx context.Context, vmID, holder string, reset bool) error { + _, err := ingress.ObjectSend[ReleaseRequest](c.ingress, EnvironmentObject, vmID, "ReleaseChain"). + Send(ctx, ReleaseRequest{Holder: holder, Reset: reset}) return err } diff --git a/internal/durable/server.go b/internal/durable/server.go index d2b9b8d..25b6e7d 100644 --- a/internal/durable/server.go +++ b/internal/durable/server.go @@ -99,8 +99,8 @@ func NewEndpoint(cfg Config, fleet Fleet, logger *slog.Logger) *server.Restate { return holdChain(ctx, fleet, req.Lease) })). Handler("ReleaseChain", restate.NewObjectHandler( - func(ctx restate.ObjectContext, reset bool) (restate.Void, error) { - return restate.Void{}, releaseChain(ctx, fleet, reset) + func(ctx restate.ObjectContext, req ReleaseRequest) (restate.Void, error) { + return restate.Void{}, releaseChain(ctx, fleet, req) })). Handler("RetireChain", restate.NewObjectHandler( func(ctx restate.ObjectContext, _ restate.Void) (restate.Void, error) { diff --git a/internal/durable/snapshot_test.go b/internal/durable/snapshot_test.go index 8dddfa6..5735d10 100644 --- a/internal/durable/snapshot_test.go +++ b/internal/durable/snapshot_test.go @@ -469,7 +469,21 @@ func TestHoldChainKeepsTheTickOff(t *testing.T) { if got := fleet.steps(); got != "" { t.Fatalf("held tick ran steps %s", got) } - if err := client.ReleaseChain(ctx, vm, false); err != nil { + // a second hold is refused while the first is live. + if again, err := client.HoldChain(ctx, vm, time.Hour); err != nil || !again.Busy { + t.Fatalf("second hold = %+v, %v; want busy", again, err) + } + // a release naming another hold changes nothing. + if err := client.ReleaseChain(ctx, vm, "not-the-holder", false); err != nil { + t.Fatal(err) + } + if err := client.Checkpoint(ctx, vm); err != nil { + t.Fatal(err) + } + if got := fleet.steps(); got != "" { + t.Fatalf("tick ran steps %s after a stranger's release", got) + } + if err := client.ReleaseChain(ctx, vm, hold.Holder, false); err != nil { t.Fatal(err) } if err := client.Checkpoint(ctx, vm); err != nil { diff --git a/internal/orchestrator/checkpoints_durable.go b/internal/orchestrator/checkpoints_durable.go index a568b84..0a0ca5e 100644 --- a/internal/orchestrator/checkpoints_durable.go +++ b/internal/orchestrator/checkpoints_durable.go @@ -50,8 +50,10 @@ type ChainMove struct { // ChainHold is what HoldChain hands a migrate. type ChainHold struct { - Busy bool `json:"busy,omitempty"` - Chain ChainState `json:"chain"` + Busy bool `json:"busy,omitempty"` + // Holder names the hold, for its release. + Holder string `json:"holder,omitempty"` + Chain ChainState `json:"chain"` } // ChainMoveResult is the end of a chain's move, sent by the Artifacts move @@ -232,29 +234,30 @@ func (fm *FleetManager) ChainAdvanced(ctx context.Context, vmID string, chain Ch } // holdDurableChain holds vmID's chain on restate for a delta migrate and -// returns a copy of it, or nil when it cannot be held: restate refused, or a -// tick's copy is still running. a migrate without the hold takes a full seed +// returns a copy of it and the hold's id, or nil when it cannot be held: +// restate refused, a tick's copy is still running, or another migrate holds +// it. a migrate without the hold takes a full seed // instead, which only costs the chain its next diff. -func (fm *FleetManager) holdDurableChain(ctx context.Context, d DurableSnapshots, vmID string) *checkpointChain { +func (fm *FleetManager) holdDurableChain(ctx context.Context, d DurableSnapshots, vmID string) (*checkpointChain, string) { hold, err := d.HoldChain(ctx, vmID, chainHoldLease) if err != nil { fm.logger.WarnContext(ctx, "hold checkpoint chain failed; migrating without it", "vm", vmID, "err", err) - return nil + return nil, "" } if hold.Busy { - fm.logger.InfoContext(ctx, "checkpoint chain copy in flight; migrating without it", "vm", vmID) - return nil + fm.logger.InfoContext(ctx, "checkpoint chain busy (a copy in flight or another migrate); migrating without it", "vm", vmID) + return nil, "" } c := hold.Chain - return &checkpointChain{standby: c.Standby, head: c.Head, base: c.Base, bytes: c.Bytes} + return &checkpointChain{standby: c.Standby, head: c.Head, base: c.Base, bytes: c.Bytes}, hold.Holder } // releaseDurableChain ends a migrate's hold. reset forgets the chain, whose // base the migrate consumed or put out of step. -func (fm *FleetManager) releaseDurableChain(d DurableSnapshots, vmID string, reset bool) { +func (fm *FleetManager) releaseDurableChain(d DurableSnapshots, vmID, holder string, reset bool) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - if err := d.ReleaseChain(ctx, vmID, reset); err != nil { + if err := d.ReleaseChain(ctx, vmID, holder, reset); err != nil { fm.logger.Warn("release checkpoint chain failed; the hold lapses on its lease", "vm", vmID, "err", err) } } diff --git a/internal/orchestrator/migrate.go b/internal/orchestrator/migrate.go index b044869..36f4126 100644 --- a/internal/orchestrator/migrate.go +++ b/internal/orchestrator/migrate.go @@ -195,12 +195,13 @@ func (fm *FleetManager) migrateVM(ctx context.Context, vmID string, opts Migrate if d := fm.durableSnapshotsClient(); delta && d != nil { // on restate the chain lives on the vm's Environment object, and // the hold is what keeps the checkpoint loop off it. - chain = fm.holdDurableChain(ctx, d, vmID) + var holder string + chain, holder = fm.holdDurableChain(ctx, d, vmID) if chain == nil { delta = false } else { held := chain.base - defer func() { fm.releaseDurableChain(d, vmID, held != "" && chain.base == "") }() + defer func() { fm.releaseDurableChain(d, vmID, holder, held != "" && chain.base == "") }() } } else if delta { // held for the whole migrate so the checkpoint loop cannot take a live diff --git a/internal/orchestrator/snapshot_durable.go b/internal/orchestrator/snapshot_durable.go index bd10f67..4c2e518 100644 --- a/internal/orchestrator/snapshot_durable.go +++ b/internal/orchestrator/snapshot_durable.go @@ -71,11 +71,11 @@ type DurableSnapshots interface { Checkpoint(ctx context.Context, vmID string) error // HoldChain keeps the checkpoint loop off vmID's chain for lease, or // until ReleaseChain, and returns the chain. Busy is set while a tick's - // copy is still running; nothing is held then. + // copy is still running or another hold is live; nothing is held then. HoldChain(ctx context.Context, vmID string, lease time.Duration) (ChainHold, error) - // ReleaseChain ends a hold. reset forgets the chain, whose base the - // holder consumed or put out of step. - ReleaseChain(ctx context.Context, vmID string, reset bool) error + // ReleaseChain ends the hold named holder. reset forgets the chain, + // whose base the holder consumed or put out of step. + ReleaseChain(ctx context.Context, vmID, holder string, reset bool) error // SendRetireChain enqueues the removal of what a gone vm's chain holds. SendRetireChain(ctx context.Context, vmID string) error // SendCollectSnapshot enqueues the deletion of a snapshot retention gc @@ -147,7 +147,8 @@ func (fm *FleetManager) sendCollectSnapshot(ctx context.Context, d DurableSnapsh // SnapshotCollect deletes a snapshot retention gc marked deleting: its host // artifact, then its row. a row that is gone or no longer deleting was dealt -// with already. a host that no longer has the artifact is not an error. +// with already, and a pinned one is handed back. a host that no longer has +// the artifact is not an error. func (fm *FleetManager) SnapshotCollect(ctx context.Context, snapshotID string) error { record, found, err := fm.lookupSnapshotRecord(ctx, snapshotID) if err != nil { @@ -156,6 +157,18 @@ func (fm *FleetManager) SnapshotCollect(ctx context.Context, snapshotID string) if !found || record.State != SnapshotStateDeleting { return nil } + // the collection may have queued behind other ops on the vm while a move + // pinned the snapshot. a pinned one goes back to ready, and a later sweep + // marks it again once the pin is released. + if fm.snapshotPinned(ctx, snapshotID) { + record.State = SnapshotStateReady + record.UpdatedAt = time.Now() + if err := fm.upsertSnapshotRecord(ctx, record); err != nil { + return fmt.Errorf("unmark pinned snapshot %s: %w", snapshotID, err) + } + fm.logger.InfoContext(ctx, "snapshot gc skipped a pinned snapshot", "snapshot", snapshotID) + return nil + } if err := fm.deleteSnapshotArtifact(ctx, record); err != nil && !IsNotFound(err) { return err } diff --git a/internal/orchestrator/snapshot_durable_test.go b/internal/orchestrator/snapshot_durable_test.go index f6697c4..ea4dd97 100644 --- a/internal/orchestrator/snapshot_durable_test.go +++ b/internal/orchestrator/snapshot_durable_test.go @@ -53,7 +53,7 @@ func (f *fakeDurableSnapshots) HoldChain(_ context.Context, vmID string, _ time. return f.hold, f.holdErr } -func (f *fakeDurableSnapshots) ReleaseChain(_ context.Context, _ string, reset bool) error { +func (f *fakeDurableSnapshots) ReleaseChain(_ context.Context, _, _ string, reset bool) error { f.mu.Lock() defer f.mu.Unlock() f.releases = append(f.releases, reset) @@ -241,6 +241,24 @@ func TestReconcileSnapshots_durableSendsCollect(t *testing.T) { t.Fatalf("%d checkpoints, want both still there", n) } + // a move that pinned it after the marking keeps it, back in ready. + da := newFakeDurableArtifacts() + fm.SetDurableArtifacts(da) + if err := da.PinArtifacts(ctx, "move:x", 0, first.SnapshotID); err != nil { + t.Fatal(err) + } + if err := fm.SnapshotCollect(ctx, first.SnapshotID); err != nil { + t.Fatalf("collect pinned: %v", err) + } + if rec, _ := fm.GetSnapshotByID(ctx, first.SnapshotID); rec.State != SnapshotStateReady { + t.Fatalf("pinned row %s after collect, want ready", rec.State) + } + _ = da.ReleaseArtifacts(ctx, "move:x", first.SnapshotID) + rec.State = SnapshotStateDeleting + if err := fm.upsertSnapshotRecord(ctx, rec); err != nil { + t.Fatal(err) + } + if err := fm.SnapshotCollect(ctx, first.SnapshotID); err != nil { t.Fatalf("collect: %v", err) } From 179dc8b6f7e0f84033c4cd30fc44d18373640902 Mon Sep 17 00:00:00 2001 From: andrewn6 <59238070+andrewn6@users.noreply.github.com> Date: Sun, 15 Nov 2026 00:33:41 -0500 Subject: [PATCH 6/6] fix: refuse to pull a move source that snapshot gc is deleting --- internal/orchestrator/artifacts_durable.go | 10 ++++++++++ internal/orchestrator/artifacts_durable_test.go | 15 +++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/internal/orchestrator/artifacts_durable.go b/internal/orchestrator/artifacts_durable.go index 640d8ea..749a26e 100644 --- a/internal/orchestrator/artifacts_durable.go +++ b/internal/orchestrator/artifacts_durable.go @@ -245,6 +245,16 @@ func (fm *FleetManager) ArtifactPull(ctx context.Context, job ArtifactMoveJob) ( if fm.artifactMover == nil { return ArtifactMoved{}, fmt.Errorf("%w: no artifact mover is configured", ErrArtifactImmovable) } + // the move pinned its source before this step, and snapshot gc checks + // pins before it deletes. reading the row after the pin closes the gap + // where gc checked first: a source it is deleting is not read. + source, found, err := fm.lookupSnapshotRecord(ctx, job.SnapshotID) + if err != nil { + return ArtifactMoved{}, err + } + if found && source.State == SnapshotStateDeleting { + return ArtifactMoved{}, fmt.Errorf("%w: snapshot %s is being deleted", ErrArtifactImmovable, job.SnapshotID) + } from, err := fm.artifactEndpoint(job.SourceHostID) if err != nil { return ArtifactMoved{}, err diff --git a/internal/orchestrator/artifacts_durable_test.go b/internal/orchestrator/artifacts_durable_test.go index a3ca1d4..f5bed4b 100644 --- a/internal/orchestrator/artifacts_durable_test.go +++ b/internal/orchestrator/artifacts_durable_test.go @@ -171,6 +171,21 @@ func TestArtifactSteps(t *testing.T) { if _, err := fm.ArtifactPull(ctx, gone); !PermanentArtifactError(err) { t.Fatalf("pull from an unregistered host: %v, want a permanent error", err) } + + // a source gc marked deleting after the move planned is not read. + deleting := seedArtifactRecord(t, fm, "art-del", "tenant-1", "h-a", "d3") + deleting.State = SnapshotStateDeleting + if err := fm.upsertSnapshotRecord(ctx, deleting); err != nil { + t.Fatalf("mark deleting: %v", err) + } + doomed := job + doomed.SnapshotID, doomed.SourceSnapshotID, doomed.LocalID = "art-del", "art-del", "art-del-copy" + if _, err := fm.ArtifactPull(ctx, doomed); !PermanentArtifactError(err) { + t.Fatalf("pull of a deleting source: %v, want a permanent error", err) + } + if n := len(mover.calls()); n != 1 { + t.Fatalf("mover ran %d times, want the deleting source never pulled", n) + } } // TestArtifactDiscard deletes an unrecorded copy and keeps a recorded one.