From a9677a15b6c8bdecda73474916f7456e2f12ea9f Mon Sep 17 00:00:00 2001 From: ktechmidas <9920871+ktechmidas@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:44:54 +0300 Subject: [PATCH 1/4] feat(provision): plan the reuse of a destroyed network's name A network key (account/region/name) was bound to its first plan for ever, so a devnet that had been deployed and destroyed could never be created again under its name. A plan can now succeed a retired one. Plan gains `succeeds`, the ID of the retired plan it replaces, covered by the plan ID: the successor of an unchanged network definition still has its own ID, EC2 client tokens and `dashnet:plan` tags. A first plan serializes exactly as before. Record.Retired is the only proof accepted that a network is gone: the phase `addresses-released`, which ReleaseAddresses writes after it has seen every journaled instance terminated (or purged) under the plan's ownership and every address absent by its exact ID, with every target resolved. A plan without IPAM addresses has no such proof and is never retired. Execute hands a successor plan to a Superseder store instead of Acquire, so only provision can take a name over. The successor's operation starts empty and keeps a `predecessor` entry: the retired plan ID, where its journal is archived, and its instance IDs. The live-scope check is unchanged except for one case. EC2 lists a terminated instance for about an hour; an instance is now skipped if it is terminated, carries the predecessor's plan tag and is one the predecessor journaled. Anything else under the network tag still blocks, and nothing is adopted. Co-Authored-By: Claude Fable 5.2 --- internal/provision/bootstrap.go | 2 +- internal/provision/ec2.go | 10 +- internal/provision/ipam_test.go | 2 +- internal/provision/plan.go | 39 +++- internal/provision/preflight.go | 8 +- internal/provision/reuse_test.go | 329 +++++++++++++++++++++++++++++++ internal/provision/runner.go | 103 +++++++++- 7 files changed, 480 insertions(+), 13 deletions(-) create mode 100644 internal/provision/reuse_test.go diff --git a/internal/provision/bootstrap.go b/internal/provision/bootstrap.go index b7d849a..43c77d6 100644 --- a/internal/provision/bootstrap.go +++ b/internal/provision/bootstrap.go @@ -72,7 +72,7 @@ func RunningTargets(ctx context.Context, p Plan, r Record, cloud EC2) (map[strin if r.Phase != "compute-ready" { return nil, errors.New("bootstrap requires completed EC2 provisioning") } - live, err := discover(ctx, p, cloud) + live, err := discover(ctx, p, cloud, r.Predecessor) if err != nil { return nil, err } diff --git a/internal/provision/ec2.go b/internal/provision/ec2.go index a8088b1..ab454e4 100644 --- a/internal/provision/ec2.go +++ b/internal/provision/ec2.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "sort" "strconv" "time" @@ -37,7 +38,11 @@ func launch(ctx context.Context, p Plan, t Target, c EC2) (string, error) { return aws.ToString(out.Instances[0].InstanceId), nil } -func discover(ctx context.Context, p Plan, c EC2) (map[string]types.Instance, error) { +// discover returns this plan's live instances and refuses everything else that +// carries the network tag. The one exception is the retired predecessor's own +// instances: EC2 lists a terminated instance for about an hour, and one that +// the predecessor journaled, still tagged with its plan, can never run again. +func discover(ctx context.Context, p Plan, c EC2, retired *Predecessor) (map[string]types.Instance, error) { input := &ec2.DescribeInstancesInput{Filters: []types.Filter{{Name: aws.String("tag:" + p.Network.AWS.NetworkTagKey), Values: []string{p.Network.Metadata.Name}}}} targets := map[string]Target{} for _, t := range p.Targets { @@ -67,6 +72,9 @@ func discover(ctx context.Context, p Plan, c EC2) (map[string]types.Instance, er } tags[k] = aws.ToString(tag.Value) } + if retired != nil && instance.State != nil && instance.State.Name == types.InstanceStateNameTerminated && tags["dashnet:plan"] == retired.PlanID && slices.Contains(retired.Instances, aws.ToString(instance.InstanceId)) { + continue + } t, ok := targets[tags["dashnet:node"]] if !ok { return nil, errors.New("network tag includes an unmanaged or unexpected target; refusing legacy adoption") diff --git a/internal/provision/ipam_test.go b/internal/provision/ipam_test.go index ebe150d..d2b9bea 100644 --- a/internal/provision/ipam_test.go +++ b/internal/provision/ipam_test.go @@ -275,7 +275,7 @@ func TestIPAMRejectsWrongPoolAndDrift(t *testing.T) { func TestNewPublicProvisionRequiresPoolButHistoricalPlansRemainReadable(t *testing.T) { n := testutil.ProvisionNetwork(t) n.AWS.Provision.PublicIPv4 = true - p := build(n, map[string]string{"arm64": "/dev/sda1"}) + p := build(n, map[string]string{"arm64": "/dev/sda1"}, "") if err := p.Validate(); err != nil { t.Fatal("historical plan no longer readable", err) } diff --git a/internal/provision/plan.go b/internal/provision/plan.go index 1771a94..f39618b 100644 --- a/internal/provision/plan.go +++ b/internal/provision/plan.go @@ -16,10 +16,15 @@ import ( const ManagedBy = "dash-network-go" +// Succeeds is empty for the first plan under a network key. A plan that reuses +// the name of a destroyed network names the retired plan it replaces: the ID +// covers it, so every client token, tag and journal condition of the new fleet +// differs from the old one's, even when nothing else in the network changed. type Plan struct { APIVersion string `json:"apiVersion"` Kind string `json:"kind"` ID string `json:"id"` + Succeeds string `json:"succeeds,omitempty"` Network spec.Network `json:"network"` Targets []Target `json:"targets"` } @@ -40,8 +45,8 @@ func digest(v any) string { return hex.EncodeToString(sum[:]) } -func build(n spec.Network, roots map[string]string) Plan { - p := Plan{APIVersion: spec.Version, Kind: "EC2ProvisionPlan", Network: n, Targets: []Target{}} +func build(n spec.Network, roots map[string]string, succeeds string) Plan { + p := Plan{APIVersion: spec.Version, Kind: "EC2ProvisionPlan", Succeeds: succeeds, Network: n, Targets: []Target{}} for _, g := range n.Nodes { for i := 1; i <= g.Count; i++ { p.Targets = append(p.Targets, Target{Name: fmt.Sprintf("%s-%03d", g.Name, i), Group: g.Name, Role: g.Role, Architecture: g.Architecture, InstanceType: g.InstanceType, AMI: n.AWS.Provision.AMIs[g.Architecture].ID, RootDevice: roots[g.Architecture]}) @@ -69,17 +74,43 @@ func (p Plan) Validate() error { } roots[t.Architecture] = t.RootDevice } - if !reflect.DeepEqual(p, build(p.Network, roots)) { + if p.Succeeds != "" && !hex64.MatchString(p.Succeeds) { + return errors.New("provision plan succeeds an invalid plan ID") + } + if !reflect.DeepEqual(p, build(p.Network, roots, p.Succeeds)) { return errors.New("provision plan is altered, incomplete, or has an invalid ID; regenerate it") } return nil } +// Successor is p as the plan that replaces the retired plan with the given ID +// under the same network key. Only the journal decides whether that plan is +// retired: see Record.Retired and the journal's Supersede. +func (p Plan) Successor(retired string) (Plan, error) { + if !hex64.MatchString(retired) { + return Plan{}, errors.New("a successor plan needs the retired plan's ID") + } + if err := p.Validate(); err != nil { + return Plan{}, err + } + roots := map[string]string{} + for _, t := range p.Targets { + roots[t.Architecture] = t.RootDevice + } + next := build(p.Network, roots, retired) + return next, next.Validate() +} + // Key excludes generation: a changed generation cannot bypass the network lock -// or silently replace a previous fleet. This first stage never rotates records. +// or silently replace a previous fleet. A key is reused only by a plan that +// succeeds the retired plan the journal holds under it. func (p Plan) Key() string { return p.Network.AWS.AccountID + "/" + p.Network.AWS.Region + "/" + p.Network.Metadata.Name } + +// ArchiveKey is where the journal keeps this plan's operation once its network +// key has been reused. A network name cannot contain '#'. +func (p Plan) ArchiveKey() string { return p.Key() + "#retired#" + p.ID } func (p Plan) Token(t Target) string { return digest([]string{p.ID, t.Name}) } func (p Plan) Tags(t Target) map[string]string { return map[string]string{ diff --git a/internal/provision/preflight.go b/internal/provision/preflight.go index c7a4080..36b06ea 100644 --- a/internal/provision/preflight.go +++ b/internal/provision/preflight.go @@ -46,6 +46,12 @@ func arch(a string) types.ArchitectureType { // Prepare is read-only. Execute repeats these checks against the immutable plan. // It does not claim to verify routes, SSH trust, OS preparation, or chain health. func Prepare(ctx context.Context, n spec.Network, identity inventory.STS, cloud EC2) (Plan, error) { + return prepare(ctx, n, "", identity, cloud) +} + +// prepare builds the plan that succeeds the given retired plan ID, or the first +// plan under the network key when that ID is empty. +func prepare(ctx context.Context, n spec.Network, succeeds string, identity inventory.STS, cloud EC2) (Plan, error) { if err := n.Validate(); err != nil { return Plan{}, err } @@ -142,7 +148,7 @@ func Prepare(ctx context.Context, n spec.Network, identity inventory.STS, cloud return Plan{}, fmt.Errorf("instance type %s does not support requested architecture/EBS/HVM", g.InstanceType) } } - p := build(n, roots) + p := build(n, roots, succeeds) return p, p.Validate() } diff --git a/internal/provision/reuse_test.go b/internal/provision/reuse_test.go new file mode 100644 index 0000000..18f6418 --- /dev/null +++ b/internal/provision/reuse_test.go @@ -0,0 +1,329 @@ +package provision + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/dashpay/dash-network-go/internal/testutil" +) + +// Supersede is the in-memory counterpart of the journal's: the same decision, +// taken by the same NewSuccessor, without the archive. +func (s *memoryStore) Supersede(_ context.Context, p Plan, o string) (Record, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.owner != "" { + return Record{}, errors.New("busy") + } + if s.record.Plan.ID != p.ID { + next, err := NewSuccessor(p, s.record) + if err != nil { + return Record{}, err + } + s.record = next + } + s.owner = o + return copyRecord(s.record), nil +} + +// plainStore is a journal that cannot hand a name over. +type plainStore struct{ Store } + +// destroyed provisions an IPAM fleet, terminates it and retires its addresses, +// as the status console's delete does. EC2 still lists the terminated instances. +func destroyed(t *testing.T) (Plan, *addressCloud, *memoryStore) { + t.Helper() + p, c, s := ipamSetup(t) + if r, err := ipamRun(p, c, s); err != nil || r.Phase != "compute-ready" { + t.Fatal(err) + } + terminateFixture(c) + if r, err := ipamRelease(p, c, s); err != nil || r.Phase != "addresses-released" || len(c.addresses) != 0 { + t.Fatal("fleet was not retired", err) + } + if err := s.record.Retired(); err != nil { + t.Fatal("retired operation not recognised", err) + } + return p, c, s +} + +func TestDestroyedNetworkNameIsReusedBySuccessorPlan(t *testing.T) { + p, c, s := destroyed(t) + ctx := context.Background() + identity := testutil.Identity{Account: p.Network.AWS.AccountID} + // The retired plan itself, and the same network planned afresh, stay refused. + if _, err := ipamRun(p, c, s); err == nil || !strings.Contains(err.Error(), "allocation retired") { + t.Fatal("retired plan provisioned again", err) + } + again, err := Prepare(ctx, p.Network, identity, c) + if err != nil || again.ID != p.ID { + t.Fatal("an unchanged network no longer plans to the same ID", err) + } + next, err := p.Successor(p.ID) + if err != nil || next.ID == p.ID || next.Succeeds != p.ID || next.Key() != p.Key() || next.Validate() != nil { + t.Fatal("invalid successor plan", err) + } + // Nothing about the chain changed: only the plan identity did. + if next.Network.Metadata.Name != p.Network.Metadata.Name || next.Network.Chain != p.Network.Chain || len(next.Targets) != len(p.Targets) { + t.Fatal("successor changed the network") + } + for j, target := range p.Targets { + if next.Targets[j] != target || next.Token(target) == p.Token(target) || next.Tags(target)["dashnet:plan"] == p.Tags(target)["dashnet:plan"] { + t.Fatal("successor would repeat the retired fleet's EC2 identity") + } + } + if _, err = Execute(ctx, next, identity, c, plainStore{s}, "owner", "test", time.Millisecond, nil); err == nil || s.record.Plan.ID != p.ID { + t.Fatal("a journal without succession handed the name over", err) + } + old := len(c.Instances) + r, err := Execute(ctx, next, identity, c, s, "owner", "test", time.Millisecond, nil) + if err != nil || r.Phase != "compute-ready" || r.Plan.ID != next.ID || len(c.Instances) != old+len(p.Targets) || len(c.addresses) != len(p.Targets) { + t.Fatal("successor did not provision past the terminated predecessor", err) + } + if r.Predecessor == nil || r.Predecessor.PlanID != p.ID || r.Predecessor.Archive != p.ArchiveKey() || len(r.Predecessor.Instances) != len(p.Targets) || r.Bootstrap != nil || r.Deployment != nil || r.Runtime != nil || r.Upgrade != nil { + t.Fatal("successor operation is not fresh or lost its predecessor") + } + for _, target := range next.Targets { + n := r.Nodes[target.Name] + for _, id := range r.Predecessor.Instances { + if n.InstanceID == id { + t.Fatal("successor adopted a retired instance") + } + } + } + // A resume, and the later stages, still see the terminated predecessor. + if _, err = Execute(ctx, next, identity, c, s, "owner", "test", time.Millisecond, nil); err != nil || len(c.Instances) != old+len(p.Targets) { + t.Fatal("successor resume relaunched or failed", err) + } + if live, err := RunningTargets(ctx, next, s.record, c); err != nil || len(live) != len(next.Targets) { + t.Fatal("bootstrap cannot bind the successor's targets", err) + } + // The successor can itself be destroyed and succeeded. + for j := range c.Instances { + c.Instances[j].State = &types.InstanceState{Name: types.InstanceStateNameTerminated} + } + for j := range c.addresses { + c.addresses[j].AssociationId, c.addresses[j].NetworkInterfaceId, c.addresses[j].InstanceId = nil, nil, nil + } + if r, err = ipamRelease(next, c, s); err != nil || r.Phase != "addresses-released" || r.Retired() != nil { + t.Fatal("successor was not retired", err) + } + third, err := next.Successor(next.ID) + if err != nil || third.ID == next.ID || third.ID == p.ID { + t.Fatal("second successor repeats an earlier plan", err) + } + if _, err = NewSuccessor(third, s.record); err != nil { + t.Fatal(err) + } +} + +func TestSuccessorNeverAdoptsOrIgnoresAnythingButItsTerminatedPredecessor(t *testing.T) { + for _, change := range []string{"shutting-down", "stopped", "running", "unjournaled", "other-plan", "foreign-node", "live-foreign"} { + t.Run(change, func(t *testing.T) { + p, c, s := destroyed(t) + next, err := p.Successor(p.ID) + if err != nil { + t.Fatal(err) + } + retag := func(i *types.Instance, key, value string) { + tags := []types.Tag{} + for _, tag := range i.Tags { + if aws.ToString(tag.Key) == key { + tag.Value = aws.String(value) + } + tags = append(tags, tag) + } + i.Tags = tags + } + switch change { + case "shutting-down": + c.Instances[0].State = &types.InstanceState{Name: types.InstanceStateNameShuttingDown} + case "stopped": + c.Instances[0].State = &types.InstanceState{Name: types.InstanceStateNameStopped} + case "running": + c.Instances[0].State = &types.InstanceState{Name: types.InstanceStateNameRunning} + case "unjournaled": + // Terminated and tagged like the predecessor, but not an instance it recorded. + extra := c.Instances[0] + extra.InstanceId = aws.String("i-0000beef") + c.Instances = append(c.Instances, extra) + case "other-plan": + retag(&c.Instances[0], "dashnet:plan", strings.Repeat("a", 64)) + case "foreign-node": + // One of the predecessor's instances is exempt; a renamed live one is not. + c.Instances[0].State = &types.InstanceState{Name: types.InstanceStateNameRunning} + retag(&c.Instances[0], "dashnet:node", "legacy-1") + case "live-foreign": + c.Instances = append(c.Instances, types.Instance{InstanceId: aws.String("i-0000f00d"), State: &types.InstanceState{Name: types.InstanceStateNameRunning}, Tags: []types.Tag{{Key: aws.String(p.Network.AWS.NetworkTagKey), Value: aws.String(p.Network.Metadata.Name)}}}) + } + before := len(c.Instances) + r, err := ipamRun(next, c, s) + if err == nil || len(c.Instances) != before || len(c.Requests) != len(p.Targets) || len(c.addresses) != 0 { + t.Fatal("successor launched beside an unexplained instance", err) + } + if r.Phase != "interrupted" || s.record.Plan.ID != next.ID || s.owner != "" { + t.Fatal("refusal did not leave an inspectable, unclaimed operation", r.Phase) + } + }) + } +} + +func TestOnlyARetiredOperationHasASuccessor(t *testing.T) { + p, c, s := ipamSetup(t) + if _, err := ipamRun(p, c, s); err != nil { + t.Fatal(err) + } + next, err := p.Successor(p.ID) + if err != nil { + t.Fatal(err) + } + // A running network: every phase short of released addresses is refused. + for _, phase := range []string{"pending", "provisioning", "interrupted", "compute-ready"} { + r := copyRecord(s.record) + r.Phase = phase + if phase == "pending" { + for name := range r.Nodes { + r.Nodes[name] = NodeProgress{Phase: "pending", EC2State: "unknown"} + } + } + if err := r.Validate(p); err != nil { + t.Fatal(phase, err) + } + if r.Retired() == nil { + t.Fatal("unretired operation accepted in phase", phase) + } + if _, err := NewSuccessor(next, r); err == nil { + t.Fatal("unretired operation succeeded in phase", phase) + } + } + if _, err := ipamRun(next, c, s); err == nil || s.record.Plan.ID != p.ID || len(c.Requests) != len(p.Targets) { + t.Fatal("a live network's name was handed over", err) + } + terminateFixture(c) + if _, err := ipamRelease(p, c, s); err != nil { + t.Fatal(err) + } + retired := copyRecord(s.record) + if _, err := NewSuccessor(next, retired); err != nil { + t.Fatal(err) + } + // A plan naming another predecessor, or none, is not this operation's successor. + other, err := p.Successor(strings.Repeat("a", 64)) + if err != nil { + t.Fatal(err) + } + for _, wrong := range []Plan{p, other} { + if _, err := NewSuccessor(wrong, retired); err == nil { + t.Fatal("operation succeeded by a plan that does not name it") + } + } + // Retirement is exactly the proof ReleaseAddresses leaves, nothing weaker. + for _, change := range []string{"address", "instance", "phase"} { + r := copyRecord(retired) + name := p.Targets[0].Name + n := r.Nodes[name] + switch change { + case "address": + n.Address.Phase = "releasing" + case "instance": + n.InstanceID = "" + case "phase": + r.Phase = "compute-ready" + } + r.Nodes[name] = n + if r.Retired() == nil { + t.Fatal("weaker retirement proof accepted:", change) + } + } + // A successor's own journal must keep, and match, its predecessor. + fresh, err := NewSuccessor(next, retired) + if err != nil { + t.Fatal(err) + } + for _, change := range []string{"lost", "plan", "archive", "instances", "unsorted"} { + r := copyRecord(fresh) + switch change { + case "lost": + r.Predecessor = nil + case "plan": + r.Predecessor.PlanID = strings.Repeat("a", 64) + case "archive": + r.Predecessor.Archive = p.Key() + case "instances": + r.Predecessor.Instances = nil + case "unsorted": + r.Predecessor.Instances = []string{"i-2", "i-1"} + } + if r.Validate(next) == nil { + t.Fatal("invalid predecessor accepted:", change) + } + } + first := NewRecord(p) + first.Predecessor = fresh.Predecessor + if first.Validate(p) == nil { + t.Fatal("a first plan's operation claimed a predecessor") + } +} + +func TestPrivatePlanIsNeverRetired(t *testing.T) { + // No IPAM addresses, so no address retirement, so no proof the fleet is gone. + p, c, s := setup(t) + if _, err := run(context.Background(), p, c, s, "owner"); err != nil { + t.Fatal(err) + } + for _, phase := range []string{"compute-ready", "addresses-released"} { + r := copyRecord(s.record) + r.Phase = phase + if r.Retired() == nil { + t.Fatal("a plan without IPAM addresses was treated as retired in phase", phase) + } + } + next, err := p.Successor(p.ID) + if err != nil { + t.Fatal(err) + } + if _, err = run(context.Background(), next, c, s, "owner"); err == nil || s.record.Plan.ID != p.ID { + t.Fatal("private network name handed over without proof", err) + } + if _, err = p.Successor("not-a-plan"); err == nil { + t.Fatal("invalid predecessor ID accepted") + } + tampered := next + tampered.Succeeds = strings.Repeat("b", 64) + if tampered.Validate() == nil { + t.Fatal("plan ID does not cover its predecessor") + } +} + +func TestFirstPlansAndOperationsAreSerializedAsBefore(t *testing.T) { + // Existing plan IDs are digests of this JSON and existing journals are + // decoded strictly: a first plan and its operation must not gain a field. + p, _, _ := setup(t) + plan, err := json.Marshal(p) + if err != nil || strings.Contains(string(plan), "succeeds") { + t.Fatal("a first plan serializes a succession", err) + } + record, err := json.Marshal(NewRecord(p)) + if err != nil || strings.Contains(string(record), "predecessor") || strings.Contains(string(record), "succeeds") { + t.Fatal("a first operation serializes a predecessor", err) + } + unsalted := p + unsalted.ID = "" + if digest(unsalted) != p.ID { + t.Fatal("a first plan's ID is no longer the digest of its original fields") + } + next, err := p.Successor(p.ID) + if err != nil { + t.Fatal(err) + } + plan, err = json.Marshal(next) + if err != nil || !strings.Contains(string(plan), `"succeeds":"`+p.ID+`"`) { + t.Fatal("a successor plan does not state what it succeeds", err) + } +} diff --git a/internal/provision/runner.go b/internal/provision/runner.go index 2dab04a..484e5d2 100644 --- a/internal/provision/runner.go +++ b/internal/provision/runner.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "sort" "time" "github.com/dashpay/dash-network-go/internal/inventory" @@ -37,6 +38,78 @@ type Record struct { Deployment *DeploymentProgress `json:"deployment,omitempty"` Runtime *RuntimeState `json:"runtime,omitempty"` Upgrade *UpgradeProgress `json:"upgrade,omitempty"` + Predecessor *Predecessor `json:"predecessor,omitempty"` +} + +// Predecessor is what an operation keeps of the retired one it replaced under +// the same network key: where that journal is archived, and the instances it +// had, so they are recognised for as long as EC2 still lists them terminated. +type Predecessor struct { + PlanID string `json:"planId"` + Revision int64 `json:"revision"` + Archive string `json:"archive"` + Instances []string `json:"instances"` +} + +func (d *Predecessor) validate(p Plan) error { + if d == nil { + if p.Succeeds != "" { + return errors.New("successor operation lost its predecessor") + } + return nil + } + if p.Succeeds == "" || d.PlanID != p.Succeeds || d.Revision < 1 || d.Archive != p.Key()+"#retired#"+d.PlanID || len(d.Instances) == 0 || len(d.Instances) > 100 || !sort.StringsAreSorted(d.Instances) { + return errors.New("invalid predecessor in operation journal") + } + for i, id := range d.Instances { + if id == "" || (i > 0 && id == d.Instances[i-1]) { + return errors.New("invalid predecessor instance in operation journal") + } + } + return nil +} + +// Retired reports nil only when this record, read on its own terms, proves its +// network is gone: address retirement finished, which ReleaseAddresses reaches +// only after it has seen every journaled instance terminated (or purged) under +// this plan's ownership and every address absent by its exact ID. Nothing +// weaker counts, so a plan without IPAM addresses can never be retired. +func (r Record) Retired() error { + if err := r.ValidateAddressCleanup(r.Plan); err != nil { + return err + } + if r.Phase != "addresses-released" || !r.Plan.Network.AWS.Provision.PublicIPv4 || r.Plan.Network.AWS.Provision.IPAMPoolID == "" { + return errors.New("operation is not retired: its addresses were not released") + } + for _, t := range r.Plan.Targets { + n := r.Nodes[t.Name] + if n.Phase != "present" || n.InstanceID == "" || (n.Address != nil && n.Address.Phase != "released") { + return fmt.Errorf("operation is not retired: %s is unresolved", t.Name) + } + } + return nil +} + +// NewSuccessor is the fresh operation of a plan that succeeds a retired one. +// Nothing of the old operation is carried over except its identity. +func NewSuccessor(p Plan, retired Record) (Record, error) { + if err := p.Validate(); err != nil { + return Record{}, err + } + if p.Succeeds == "" || retired.Plan.ID != p.Succeeds || retired.Plan.Key() != p.Key() { + return Record{}, errors.New("plan does not succeed this operation") + } + if err := retired.Retired(); err != nil { + return Record{}, err + } + r := NewRecord(p) + d := &Predecessor{PlanID: retired.Plan.ID, Revision: retired.Revision, Archive: retired.Plan.ArchiveKey(), Instances: []string{}} + for _, n := range retired.Nodes { + d.Instances = append(d.Instances, n.InstanceID) + } + sort.Strings(d.Instances) + r.Predecessor = d + return r, r.validateCompute(p) } func NewRecord(p Plan) Record { @@ -73,6 +146,9 @@ func (r Record) validateCompute(p Plan) error { if r.Revision < 0 || !reflect.DeepEqual(r.Plan, p) || r.APIVersion != p.APIVersion || r.Kind != "EC2ProvisionOperation" || r.ApplicationHealth != "unknown" || len(r.Nodes) != len(p.Targets) || r.CreatedAt.IsZero() || r.UpdatedAt.IsZero() { return errors.New("operation journal identity, schema, or target set mismatch") } + if err := r.Predecessor.validate(p); err != nil { + return err + } switch r.Phase { case "pending", "provisioning", "interrupted", "compute-ready", "addresses-released": default: @@ -122,6 +198,15 @@ type Store interface { Release(context.Context, Plan, string) error } +// Superseder is a Store that can hand a network key to the plan that succeeds +// the retired operation held under it. Supersede is Acquire for such a plan: +// it archives the retired operation, then replaces it with the successor's +// fresh one, claimed by this runner. Only Execute asks for it, so no later +// stage can retire a journal by accident. +type Superseder interface { + Supersede(context.Context, Plan, string) (Record, error) +} + // Execute verifies the footprint, records launch intent BEFORE each AWS request, // reconciles every target, and waits for EC2 running. EC2 running is not node or // application health. No SSH, bootstrap, registration, reset, or terminate call. @@ -132,14 +217,22 @@ func Execute(ctx context.Context, p Plan, identity inventory.STS, cloud EC2, sto if owner == "" || pollInterval <= 0 { return result, errors.New("runner ID and positive poll interval required") } - verified, verifyErr := Prepare(ctx, p.Network, identity, cloud) + verified, verifyErr := prepare(ctx, p.Network, p.Succeeds, identity, cloud) if verifyErr != nil { return result, verifyErr } if verified.ID != p.ID { return result, errors.New("live footprint differs from approved plan; regenerate and review") } - r, err := store.Acquire(ctx, p, owner) + acquire := store.Acquire + if p.Succeeds != "" { + successor, ok := store.(Superseder) + if !ok { + return result, errors.New("this journal cannot supersede a retired operation") + } + acquire = successor.Supersede + } + r, err := acquire(ctx, p, owner) if err != nil { return result, fmt.Errorf("claim operation: %w", err) } @@ -169,7 +262,7 @@ func Execute(ctx context.Context, p Plan, identity inventory.STS, cloud EC2, sto return } if r.Phase == "addresses-released" { - return result, errors.New("allocation retired; IPAM addresses have been released") + return result, errors.New("allocation retired; IPAM addresses have been released (to reuse the name, generate a new plan with provision-plan)") } for _, n := range r.Nodes { if n.Address != nil && (n.Address.Phase == "releasing" || n.Address.Phase == "released") { @@ -191,7 +284,7 @@ func Execute(ctx context.Context, p Plan, identity inventory.STS, cloud EC2, sto } // Validate the entire live scope before the first mutation. A legacy or foreign // instance with the selected network tag prevents adoption, even on target 100. - live, err := discover(ctx, p, cloud) + live, err := discover(ctx, p, cloud, r.Predecessor) if err != nil { return } @@ -239,7 +332,7 @@ func Execute(ctx context.Context, p Plan, identity inventory.STS, cloud EC2, sto } } for { - live, err = discover(ctx, p, cloud) + live, err = discover(ctx, p, cloud, r.Predecessor) if err != nil { return } From ec0d2441ad03090397e178e137fbc90579da95fc Mon Sep 17 00:00:00 2001 From: ktechmidas <9920871+ktechmidas@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:44:55 +0300 Subject: [PATCH 2/4] feat(journal): archive a retired operation and hand its key to the successor Supersede is Acquire for a plan that succeeds a retired one. It hands the network key over only if the live item is exactly the plan the new plan names, unclaimed, and retired on its own terms (decoded with its own plan, application history not revalidated, as address cleanup does). Nothing is deleted. The retired item is first copied verbatim to `#retired#` in the same table with a create-if-absent write; a repeat finds the same item, and a different item there stops the handover. The live item is then replaced by the successor's fresh, claimed operation in one PutItem conditional on the old plan ID, the revision that was read and the absence of an owner. Two racing runners: one write wins and the other falls back to the ordinary plan and claim rules. A lost response leaves a claim to inspect, like a lost Acquire response; an SDK retry of the accepted write is recognised as the runner's own. Acquire never creates a successor's operation, so bootstrap, deploy, join, upgrade and address cleanup cannot take a name over. Succession tells provision-plan which plan a network definition must be planned as: the successor of a retired operation, or the plan the journal already holds for this same network, so a lost plan file does not strand an interrupted operation. Archived reads an operation whose key was reused. No new IAM action: GetItem and PutItem on the journal table. CI proves the handover on real DynamoDB conditional expressions with DynamoDB Local, next to the existing managed-store proof. Co-Authored-By: Claude Fable 5.2 --- .github/workflows/ci.yml | 3 + internal/journal/dynamodb.go | 199 ++++++++- internal/journal/dynamodb_test.go | 47 +++ .../journal/supersede_integration_test.go | 215 ++++++++++ internal/journal/supersede_test.go | 378 ++++++++++++++++++ 5 files changed, 840 insertions(+), 2 deletions(-) create mode 100644 internal/journal/supersede_integration_test.go create mode 100644 internal/journal/supersede_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 709fe1e..1a8be92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,6 +87,9 @@ jobs: trap 'docker rm -f dashnet-journal >/dev/null' EXIT DASHNET_TEST_DYNAMODB=http://127.0.0.1:8000 DASHNET_DISPOSABLE_CI=1 \ go test -race -count=1 -v ./internal/managed -run '^TestManagedDynamoLocalClaimsAndRecovery$' + # A retired network name handed to its successor, on the same real expressions. + DASHNET_TEST_DYNAMODB=http://127.0.0.1:8000 DASHNET_DISPOSABLE_CI=1 \ + go test -race -count=1 -v ./internal/journal -run '^TestJournalDynamoLocalNameReuse$' - name: Node worker failure-path tests run: | python3 -m unittest discover -s internal/node/testdata -p 'test_*.py' diff --git a/internal/journal/dynamodb.go b/internal/journal/dynamodb.go index d89fccd..e546e59 100644 --- a/internal/journal/dynamodb.go +++ b/internal/journal/dynamodb.go @@ -93,8 +93,16 @@ func decode(item map[string]types.AttributeValue, p provision.Plan) (provision.R func decodeRecord(item map[string]types.AttributeValue, p provision.Plan, validate func(provision.Record, provision.Plan) error) (provision.Record, error) { if value(item, "Network") != p.Key() || value(item, "PlanID") != p.ID { - return provision.Record{}, errors.New("network is already bound to a different plan; no automatic reset, replacement, or state deletion") + return provision.Record{}, errors.New("network is already bound to a different plan; no automatic reset, replacement, or state deletion (a destroyed network's name is reused by a new provision-plan once its addresses are released)") } + r, err := decodePayload(item) + if err != nil { + return r, err + } + return r, validate(r, p) +} + +func decodePayload(item map[string]types.AttributeValue) (provision.Record, error) { data := value(item, "Data") if data == "" || len(data) > 300000 { return provision.Record{}, errors.New("invalid journal payload size") @@ -113,7 +121,182 @@ func decodeRecord(item map[string]types.AttributeValue, p provision.Plan, valida if !ok || revision.Value != strconv.FormatInt(r.Revision, 10) { return r, errors.New("journal revision mismatch") } - return r, validate(r, p) + return r, nil +} + +// retired decodes the live item under a network key on its own terms, whatever +// plan it belongs to, and returns it only if it proves that network is gone: +// an unclaimed operation, consistent with its own plan, whose address +// retirement finished. An item this package did not write is never retired. +func retired(item map[string]types.AttributeValue, network string) (provision.Record, error) { + if value(item, "Owner") != "" { + return provision.Record{}, errors.New("the operation has an active runner claim") + } + if len(item) != 4 { + return provision.Record{}, errors.New("the journal item is not a provision operation") + } + r, err := decodePayload(item) + if err != nil { + return r, err + } + if value(item, "Network") != network || r.Plan.Key() != network || value(item, "PlanID") != r.Plan.ID { + return r, errors.New("the journal item does not belong to this network key") + } + return r, r.Retired() +} + +// ErrNotRetired explains why the operation under a network key cannot be +// succeeded. It is advice for whoever is planning, never an authorization. +var ErrNotRetired = errors.New("the existing operation is not retired") + +// Succession tells provision-plan which plan p's network must be planned as, +// given what the journal holds under its key. p is the plan of the network +// definition alone. The answer is the ID the plan must succeed: +// +// - "" when the key is free, or holds p's own operation; +// - the retired plan's ID, with reused true, when the key holds a retired +// operation: the name is free again for the plan that succeeds it; +// - the ID the live operation's plan succeeds, when that plan is this same +// network: planning it again yields the same plan, so it can be resumed. +// +// Anything else is ErrNotRetired. Read-only, and never an authorization: +// provision reads the journal again and Supersede decides. +func (d Dynamo) Succession(ctx context.Context, p provision.Plan) (succeeds string, reused bool, err error) { + if p.Succeeds != "" { + return "", false, errors.New("succession starts from the plan of the network definition") + } + item, err := d.readItem(ctx, p) + if errors.Is(err, ErrNotFound) { + return "", false, nil + } + if err != nil { + return "", false, err + } + r, notRetired := retired(item, p.Key()) + if notRetired == nil { + return r.Plan.ID, true, nil + } + if live, decodeErr := decodePayload(item); decodeErr == nil && value(item, "PlanID") == live.Plan.ID { + same := p + if live.Plan.Succeeds != "" { + same, err = p.Successor(live.Plan.Succeeds) + } + if err == nil && same.ID == live.Plan.ID { + return live.Plan.Succeeds, false, nil + } + } + return "", false, fmt.Errorf("%w: %v", ErrNotRetired, notRetired) +} + +// Supersede is Acquire for a plan that succeeds a retired one. The key is +// handed over only if the live item is exactly the plan p names, unclaimed and +// retired. Nothing is deleted: the retired item is first copied, verbatim, to +// its archive key (create-if-absent, so a repeat is harmless), then replaced +// by p's fresh operation in one write conditional on the plan, the revision +// that was read and the absence of a claim. Of two runners, one write wins and +// the other finds an operation that is no longer the one it read. +func (d Dynamo) Supersede(ctx context.Context, p provision.Plan, owner string) (provision.Record, error) { + if owner == "" { + return provision.Record{}, errors.New("empty runner ID") + } + if p.Succeeds == "" { + return provision.Record{}, errors.New("plan does not succeed a retired operation") + } + item, err := d.readItem(ctx, p) + if errors.Is(err, ErrNotFound) { + return provision.Record{}, errors.New("plan succeeds an operation this journal does not hold; regenerate the plan") + } + if err != nil { + return provision.Record{}, err + } + if value(item, "PlanID") == p.ID { + // Already handed over to this plan, by an earlier run: resume it. + return d.Acquire(ctx, p, owner) + } + if value(item, "PlanID") != p.Succeeds { + return provision.Record{}, errors.New("network is bound to a plan other than the one this plan succeeds; regenerate the plan") + } + old, err := retired(item, p.Key()) + if err != nil { + return provision.Record{}, fmt.Errorf("%w: %v", ErrNotRetired, err) + } + r, err := provision.NewSuccessor(p, old) + if err != nil { + return provision.Record{}, err + } + data, err := encode(r) + if err != nil { + return provision.Record{}, err + } + archive := map[string]types.AttributeValue{"Network": str(old.Plan.ArchiveKey()), "PlanID": item["PlanID"], "Data": item["Data"], "Revision": item["Revision"]} + _, err = d.Client.PutItem(ctx, &dynamodb.PutItemInput{TableName: aws.String(d.Table), Item: archive, ConditionExpression: aws.String("attribute_not_exists(#network)"), ExpressionAttributeNames: map[string]string{"#network": "Network"}}) + if conditional(err) { + // An earlier attempt archived it. It must be this very operation. + var kept *dynamodb.GetItemOutput + kept, err = d.Client.GetItem(ctx, &dynamodb.GetItemInput{TableName: aws.String(d.Table), Key: map[string]types.AttributeValue{"Network": archive["Network"]}, ConsistentRead: aws.Bool(true)}) + if err == nil && (kept == nil || len(kept.Item) != len(archive) || value(kept.Item, "PlanID") != value(archive, "PlanID") || value(kept.Item, "Data") != value(archive, "Data") || !sameNumber(kept.Item["Revision"], archive["Revision"])) { + err = errors.New("the archive key holds a different operation; nothing replaced") + } + } + if err != nil { + return provision.Record{}, fmt.Errorf("archive retired operation: %w", err) + } + next := key(p) + next["PlanID"] = str(p.ID) + next["Data"] = str(data) + next["Owner"] = str(owner) + next["Revision"] = number(0) + _, err = d.Client.PutItem(ctx, &dynamodb.PutItemInput{ + TableName: aws.String(d.Table), Item: next, ConditionExpression: aws.String("#plan = :retired AND #revision = :revision AND attribute_not_exists(#owner)"), + ExpressionAttributeNames: map[string]string{"#plan": "PlanID", "#revision": "Revision", "#owner": "Owner"}, ExpressionAttributeValues: map[string]types.AttributeValue{":retired": str(p.Succeeds), ":revision": number(old.Revision)}, + }) + if err == nil { + return r, nil + } + if !conditional(err) { + // The write may have been accepted: like a lost Acquire response, the + // claim may exist. Inspect the operation; never assume it failed. + return provision.Record{}, fmt.Errorf("replace retired operation (outcome may be uncertain; inspect operation): %w", err) + } + // The item is no longer the one that was read. If it is this very write, + // accepted before a retry of it was refused, the handover is done. + // Otherwise another runner won, and the ordinary claim rules decide. + item, err = d.readItem(ctx, p) + if err != nil { + return provision.Record{}, err + } + if value(item, "PlanID") == p.ID && value(item, "Owner") == owner && value(item, "Data") == data && sameNumber(item["Revision"], number(0)) { + return r, nil + } + return d.Acquire(ctx, p, owner) +} + +// Archived reads the operation of a plan whose network key has been reused. +func (d Dynamo) Archived(ctx context.Context, p provision.Plan) (provision.Record, error) { + out, err := d.Client.GetItem(ctx, &dynamodb.GetItemInput{TableName: aws.String(d.Table), Key: map[string]types.AttributeValue{"Network": str(p.ArchiveKey())}, ConsistentRead: aws.Bool(true)}) + if err != nil { + return provision.Record{}, err + } + if out == nil || len(out.Item) == 0 { + return provision.Record{}, ErrNotFound + } + if len(out.Item) != 4 || value(out.Item, "Network") != p.ArchiveKey() || value(out.Item, "PlanID") != p.ID { + return provision.Record{}, errors.New("invalid archived operation") + } + r, err := decodePayload(out.Item) + if err != nil { + return r, err + } + if err = r.ValidateAddressCleanup(p); err != nil { + return r, err + } + return r, r.Retired() +} + +func sameNumber(a, b types.AttributeValue) bool { + x, ok := a.(*types.AttributeValueMemberN) + y, also := b.(*types.AttributeValueMemberN) + return ok && also && x.Value == y.Value } // Read is strongly consistent so a resume never relies on an eventually @@ -141,6 +324,18 @@ func (d Dynamo) Acquire(ctx context.Context, p provision.Plan, owner string) (pr if owner == "" { return provision.Record{}, errors.New("empty runner ID") } + if p.Succeeds != "" { + // A successor's operation is created only by Supersede, from provision. + // Every other stage may claim it once it exists, and nothing else. + if _, _, err := d.Read(ctx, p); err != nil { + return provision.Record{}, err + } + claimed, err := d.claim(ctx, p, owner) + if err != nil { + return provision.Record{}, err + } + return decode(claimed, p) + } r := provision.NewRecord(p) data, err := encode(r) if err != nil { diff --git a/internal/journal/dynamodb_test.go b/internal/journal/dynamodb_test.go index 204df39..10e5ace 100644 --- a/internal/journal/dynamodb_test.go +++ b/internal/journal/dynamodb_test.go @@ -3,6 +3,7 @@ package journal import ( "context" "errors" + "strings" "sync" "testing" @@ -22,6 +23,13 @@ type database struct { saves int reads int claimResponse func(*dynamodb.UpdateItemOutput) (*dynamodb.UpdateItemOutput, error) + // archived holds every item whose key is not a live network key. replace + // decides the fate of the one write that hands a retired key over: "refuse" + // (nothing written), "lose" (written, response lost) or "retry" (written, + // then the SDK's own retry of it refused). + archived map[string]map[string]types.AttributeValue + replace string + replaced int } func clone(in map[string]types.AttributeValue) map[string]types.AttributeValue { @@ -41,6 +49,12 @@ func (d *database) GetItem(_ context.Context, in *dynamodb.GetItemInput, _ ...fu if !aws.ToBool(in.ConsistentRead) { return nil, errors.New("journal read must be strongly consistent") } + if k := value(in.Key, "Network"); strings.Contains(k, "#") { + if d.archived[k] == nil { + return &dynamodb.GetItemOutput{}, nil + } + return &dynamodb.GetItemOutput{Item: clone(d.archived[k])}, nil + } return &dynamodb.GetItemOutput{Item: clone(d.item)}, nil } func (d *database) PutItem(_ context.Context, in *dynamodb.PutItemInput, _ ...func(*dynamodb.Options)) (*dynamodb.PutItemOutput, error) { @@ -49,9 +63,42 @@ func (d *database) PutItem(_ context.Context, in *dynamodb.PutItemInput, _ ...fu if d.failPut != nil { return nil, d.failPut } + if aws.ToString(in.ConditionExpression) == "#plan = :retired AND #revision = :revision AND attribute_not_exists(#owner)" { + // The handover of a retired key: one conditional replacement. + if len(in.ExpressionAttributeNames) != 3 || in.ExpressionAttributeNames["#plan"] != "PlanID" || in.ExpressionAttributeNames["#revision"] != "Revision" || in.ExpressionAttributeNames["#owner"] != "Owner" || len(in.ExpressionAttributeValues) != 2 { + return nil, errors.New("malformed handover condition") + } + if d.replace == "refuse" { + return nil, errors.New("handover write did not reach the table") + } + if d.item == nil || value(in.Item, "Network") != value(d.item, "Network") || value(d.item, "PlanID") != value(in.ExpressionAttributeValues, ":retired") || !sameNumber(d.item["Revision"], in.ExpressionAttributeValues[":revision"]) || value(d.item, "Owner") != "" { + return nil, &types.ConditionalCheckFailedException{} + } + d.item = clone(in.Item) + d.replaced++ + outcome := d.replace + d.replace = "" + switch outcome { + case "lose": + return nil, errors.New("lost successful handover response") + case "retry": + return nil, &types.ConditionalCheckFailedException{} + } + return &dynamodb.PutItemOutput{}, nil + } if aws.ToString(in.ConditionExpression) != "attribute_not_exists(#network)" || in.ExpressionAttributeNames["#network"] != "Network" { return nil, errors.New("missing create-if-absent condition") } + if k := value(in.Item, "Network"); strings.Contains(k, "#") { + if d.archived[k] != nil { + return nil, &types.ConditionalCheckFailedException{} + } + if d.archived == nil { + d.archived = map[string]map[string]types.AttributeValue{} + } + d.archived[k] = clone(in.Item) + return &dynamodb.PutItemOutput{}, nil + } if d.item != nil { return nil, &types.ConditionalCheckFailedException{} } diff --git a/internal/journal/supersede_integration_test.go b/internal/journal/supersede_integration_test.go new file mode 100644 index 0000000..8817887 --- /dev/null +++ b/internal/journal/supersede_integration_test.go @@ -0,0 +1,215 @@ +package journal + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/dashpay/dash-network-go/internal/provision" + "github.com/dashpay/dash-network-go/internal/testutil" +) + +// The handover of a retired network name, against real DynamoDB conditional +// expressions rather than the fake's reading of them. The fixed loopback +// endpoint and explicit disposable flag prevent accidental use against AWS. It +// runs in CI's ephemeral DynamoDB Local container. +func TestJournalDynamoLocalNameReuse(t *testing.T) { + endpoint := os.Getenv("DASHNET_TEST_DYNAMODB") + if endpoint == "" { + t.Skip("requires disposable DynamoDB Local") + } + if endpoint != "http://127.0.0.1:8000" || os.Getenv("DASHNET_DISPOSABLE_CI") != "1" { + t.Fatal("only explicit disposable loopback DynamoDB is supported") + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + _, fixture, p, next, cloud := retiredFixture(t) + retiredItem := clone(fixture.item) + table := fmt.Sprintf("journal-test-%d", time.Now().UnixNano()) + client := dynamodb.NewFromConfig(aws.Config{ + Region: p.Network.AWS.Region, + Credentials: aws.CredentialsProviderFunc(func(context.Context) (aws.Credentials, error) { + return aws.Credentials{AccessKeyID: "disposable", SecretAccessKey: "disposable"}, nil + }), + }, func(o *dynamodb.Options) { o.BaseEndpoint = &endpoint }) + input := &dynamodb.CreateTableInput{ + TableName: aws.String(table), BillingMode: types.BillingModePayPerRequest, + KeySchema: []types.KeySchemaElement{{AttributeName: aws.String("Network"), KeyType: types.KeyTypeHash}}, + AttributeDefinitions: []types.AttributeDefinition{{AttributeName: aws.String("Network"), AttributeType: types.ScalarAttributeTypeS}}, + } + ready, stop := context.WithTimeout(ctx, 30*time.Second) + defer stop() + for { + if _, err := client.CreateTable(ready, input); err == nil { + break + } else if ready.Err() != nil { + t.Fatal("disposable journal did not start", err) + } + select { + case <-ready.Done(): + t.Fatal("disposable journal did not start", ready.Err()) + case <-time.After(200 * time.Millisecond): + } + } + t.Cleanup(func() { + cleanup, done := context.WithTimeout(context.Background(), 10*time.Second) + defer done() + if _, err := client.DeleteTable(cleanup, &dynamodb.DeleteTableInput{TableName: aws.String(table)}); err != nil { + t.Error("disposable table cleanup", err) + } + }) + store := Dynamo{Client: client, Table: table} + get := func(network string) map[string]types.AttributeValue { + t.Helper() + out, err := client.GetItem(ctx, &dynamodb.GetItemInput{TableName: aws.String(table), Key: map[string]types.AttributeValue{"Network": str(network)}, ConsistentRead: aws.Bool(true)}) + if err != nil { + t.Fatal(err) + } + return out.Item + } + put := func(item map[string]types.AttributeValue) { + t.Helper() + if _, err := client.PutItem(ctx, &dynamodb.PutItemInput{TableName: aws.String(table), Item: item}); err != nil { + t.Fatal(err) + } + } + unchanged := func(want map[string]types.AttributeValue) bool { + got := get(p.Key()) + return len(got) == len(want) && value(got, "PlanID") == value(want, "PlanID") && value(got, "Data") == value(want, "Data") && value(got, "Owner") == value(want, "Owner") && sameNumber(got["Revision"], want["Revision"]) && len(get(p.ArchiveKey())) == 0 + } + + t.Run("refused while claimed or not retired", func(t *testing.T) { + claimed := clone(retiredItem) + claimed["Owner"] = str("another-runner") + put(claimed) + if _, err := store.Supersede(ctx, next, "runner"); err == nil || !unchanged(claimed) { + t.Fatal("claimed retired operation superseded", err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal([]byte(value(retiredItem, "Data")), &fields); err != nil { + t.Fatal(err) + } + fields["phase"] = json.RawMessage(`"compute-ready"`) + data, err := json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + running := clone(retiredItem) + running["Data"] = str(string(data)) + put(running) + if _, _, err = store.Succession(ctx, p); err != nil { + t.Fatal("a live network no longer plans as itself", err) + } + if _, err = store.Supersede(ctx, next, "runner"); err == nil || !unchanged(running) { + t.Fatal("live operation superseded", err) + } + put(retiredItem) + if _, err = store.Acquire(ctx, next, "bootstrap"); err == nil || !unchanged(retiredItem) { + t.Fatal("a stage other than provision took the name over", err) + } + }) + + n := p.Network + n.Chain.Generation++ + base, err := provision.Prepare(ctx, n, testutil.Identity{Account: n.AWS.AccountID}, cloud) + if err != nil { + t.Fatal(err) + } + rival, err := base.Successor(p.ID) + if err != nil || rival.ID == next.ID { + t.Fatal("rival successor invalid", err) + } + var winner provision.Plan + var record provision.Record + owner := "" + + t.Run("competing successors and the kept archive", func(t *testing.T) { + if succeeds, reused, err := store.Succession(ctx, p); err != nil || succeeds != p.ID || !reused { + t.Fatal("retired operation not recognised", err) + } + type result struct { + plan provision.Plan + owner string + record provision.Record + err error + } + results := make(chan result, 6) + var wg sync.WaitGroup + for j, plan := range []provision.Plan{next, rival, next, rival, next, rival} { + runner := fmt.Sprint("runner-", j) + wg.Go(func() { + r, err := store.Supersede(ctx, plan, runner) + results <- result{plan, runner, r, err} + }) + } + wg.Wait() + close(results) + wins := 0 + for r := range results { + if r.err == nil { + winner, owner, record = r.plan, r.owner, r.record + wins++ + } + } + if wins != 1 { + t.Fatal("handover was not exclusive", wins) + } + live, kept := get(p.Key()), get(p.ArchiveKey()) + if value(live, "PlanID") != winner.ID || value(live, "Owner") != owner || !sameNumber(live["Revision"], number(0)) { + t.Fatal("live item is not the winner's claimed operation") + } + if len(kept) != 4 || value(kept, "PlanID") != p.ID || value(kept, "Data") != value(retiredItem, "Data") || !sameNumber(kept["Revision"], retiredItem["Revision"]) { + t.Fatal("retired operation was not archived verbatim") + } + archived, err := store.Archived(ctx, p) + if err != nil || archived.Phase != "addresses-released" || archived.Plan.ID != p.ID { + t.Fatal("archived operation unreadable", err) + } + if record.Predecessor == nil || record.Predecessor.PlanID != p.ID || record.Predecessor.Archive != p.ArchiveKey() { + t.Fatal("successor lost its predecessor") + } + }) + + t.Run("successor is an ordinary operation and the loser stays refused", func(t *testing.T) { + loser := rival + if winner.ID == rival.ID { + loser = next + } + record.Phase, record.LastRunner = "provisioning", owner + record.Revision++ + if err := store.Save(ctx, record, owner); err != nil { + t.Fatal(err) + } + if err := store.Release(ctx, winner, owner); err != nil { + t.Fatal(err) + } + if _, err := store.Supersede(ctx, loser, "late"); err == nil || value(get(p.Key()), "PlanID") != winner.ID || value(get(p.Key()), "Owner") != "" { + t.Fatal("losing plan replaced the winner", err) + } + for _, acquire := range []func(context.Context, provision.Plan, string) (provision.Record, error){store.Supersede, store.Acquire} { + resumed, err := acquire(ctx, winner, "resume") + if err != nil || resumed.Phase != "provisioning" || resumed.Revision != 1 { + t.Fatal("successor operation did not resume", err) + } + if err = store.Release(ctx, winner, "resume"); err != nil { + t.Fatal(err) + } + } + if value(get(p.ArchiveKey()), "Data") != value(retiredItem, "Data") { + t.Fatal("archive changed after the handover") + } + if _, _, err := store.Read(ctx, p); err == nil { + t.Fatal("retired plan still reads the live operation") + } + if _, err := store.Supersede(ctx, p, "late"); err == nil { + t.Fatal("a plan that succeeds nothing superseded an operation") + } + }) +} diff --git a/internal/journal/supersede_test.go b/internal/journal/supersede_test.go new file mode 100644 index 0000000..6fcc6f5 --- /dev/null +++ b/internal/journal/supersede_test.go @@ -0,0 +1,378 @@ +package journal + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/dashpay/dash-network-go/internal/provision" + "github.com/dashpay/dash-network-go/internal/testutil" +) + +// retiredFixture is a destroyed network as the journal really holds one: the +// historical fleet of cleanupFixture, whose application history today's rules +// reject, after release-addresses retired it. next is the plan provision-plan +// then produces for the same, unchanged network definition. +func retiredFixture(t *testing.T) (Dynamo, *database, provision.Plan, provision.Plan, *cleanupCloud) { + t.Helper() + d, db, p, c := cleanupFixture(t) + ctx := context.Background() + // While the fleet has its addresses, the network plans as itself. + if succeeds, reused, err := d.Succession(ctx, p); err != nil || succeeds != "" || reused { + t.Fatal("a network that still has its addresses was offered for reuse", err) + } + if r, err := provision.ReleaseAddresses(ctx, p, testutil.Identity{Account: p.Network.AWS.AccountID}, c, d.AddressCleanup(), "destroy"); err != nil || r.Phase != "addresses-released" { + t.Fatal("fixture was not retired", err) + } + succeeds, reused, err := d.Succession(ctx, p) + if err != nil || succeeds != p.ID || !reused { + t.Fatal("retired operation not recognised", err) + } + next, err := p.Successor(succeeds) + if err != nil || next.ID == p.ID || next.Key() != p.Key() { + t.Fatal("invalid successor plan", err) + } + return d, db, p, next, c +} + +// rewrite changes the stored payload of the live item, as raw JSON. +func rewrite(t *testing.T, db *database, change func(map[string]json.RawMessage)) { + t.Helper() + var fields map[string]json.RawMessage + if err := json.Unmarshal([]byte(value(db.item, "Data")), &fields); err != nil { + t.Fatal(err) + } + change(fields) + data, err := json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + db.item["Data"] = str(string(data)) +} + +func TestRetiredNameIsArchivedAndHandedToItsSuccessor(t *testing.T) { + d, db, p, next, _ := retiredFixture(t) + ctx := context.Background() + before := clone(db.item) + // No stage but provision hands a name over, and the retired plan stays retired. + if _, err := d.Acquire(ctx, next, "bootstrap"); err == nil || value(db.item, "PlanID") != p.ID || value(db.item, "Owner") != "" || len(db.archived) != 0 { + t.Fatal("a stage other than provision took the name over", err) + } + if _, err := d.Supersede(ctx, p, "retired"); err == nil { + t.Fatal("a plan that succeeds nothing superseded an operation") + } + r, err := d.Supersede(ctx, next, "one") + if err != nil || r.Phase != "pending" || r.Revision != 0 || r.Plan.ID != next.ID || r.Predecessor == nil || r.Predecessor.PlanID != p.ID || len(r.Predecessor.Instances) != len(p.Targets) || r.Bootstrap != nil || r.Deployment != nil || r.Runtime != nil || r.Upgrade != nil { + t.Fatal("successor did not start a fresh operation", err) + } + kept := db.archived[p.ArchiveKey()] + if len(db.archived) != 1 || len(kept) != 4 || value(kept, "Network") != p.ArchiveKey() || value(kept, "PlanID") != p.ID || value(kept, "Data") != value(before, "Data") || !sameNumber(kept["Revision"], before["Revision"]) { + t.Fatal("retired operation was not archived verbatim") + } + if value(db.item, "Network") != p.Key() || value(db.item, "PlanID") != next.ID || value(db.item, "Owner") != "one" || !sameNumber(db.item["Revision"], number(0)) { + t.Fatal("live item is not the successor's claimed operation") + } + current, owner, err := d.Read(ctx, next) + if err != nil || owner != "one" || current.Predecessor.Archive != p.ArchiveKey() || current.Predecessor.Revision != r.Predecessor.Revision { + t.Fatal("successor operation unreadable", err) + } + if _, _, err = d.Read(ctx, p); err == nil { + t.Fatal("retired plan still reads the live operation") + } + archived, err := d.Archived(ctx, p) + if err != nil || archived.Phase != "addresses-released" || archived.Plan.ID != p.ID || len(archived.Nodes) != len(p.Targets) { + t.Fatal("archived operation unreadable", err) + } + if _, err = d.Archived(ctx, next); !errors.Is(err, ErrNotFound) { + t.Fatal("live operation reported as archived", err) + } + // The retired plan can no longer be cleaned up, claimed or unlocked: its key is not its own. + if _, err = d.AddressCleanup().Acquire(ctx, p, "cleanup"); err == nil || value(db.item, "Owner") != "one" { + t.Fatal("retired plan claimed its successor's operation", err) + } + // From here on it is an ordinary operation, for provision and every later stage. + r.Phase, r.LastRunner = "provisioning", "one" + r.Revision++ + if err = d.Save(ctx, r, "one"); err != nil { + t.Fatal(err) + } + if err = d.Release(ctx, next, "one"); err != nil { + t.Fatal(err) + } + for _, acquire := range []func(context.Context, provision.Plan, string) (provision.Record, error){d.Supersede, d.Acquire} { + resumed, err := acquire(ctx, next, "two") + if err != nil || resumed.Phase != "provisioning" || resumed.Revision != 1 || len(db.archived) != 1 || db.replaced != 1 { + t.Fatal("successor operation did not resume", err) + } + if err = d.Release(ctx, next, "two"); err != nil { + t.Fatal(err) + } + } + // Planning the same network again yields the same successor plan, not a new one. + succeeds, reused, err := d.Succession(ctx, p) + if err != nil || succeeds != p.ID || reused { + t.Fatal("an interrupted successor cannot be planned again", err) + } + if value(db.archived[p.ArchiveKey()], "Data") != value(before, "Data") { + t.Fatal("archive changed after the handover") + } +} + +func TestOnlyARetiredUnclaimedOperationIsSuperseded(t *testing.T) { + phase := func(name string) func(*testing.T, *database) { + return func(t *testing.T, db *database) { + rewrite(t, db, func(fields map[string]json.RawMessage) { fields["phase"] = json.RawMessage(`"` + name + `"`) }) + } + } + node := func(change func(map[string]any)) func(*testing.T, *database) { + return func(t *testing.T, db *database) { + rewrite(t, db, func(fields map[string]json.RawMessage) { + var nodes map[string]map[string]any + if err := json.Unmarshal(fields["nodes"], &nodes); err != nil { + t.Fatal(err) + } + change(nodes["wallet-001"]) + fields["nodes"], _ = json.Marshal(nodes) + }) + } + } + cases := map[string]func(*testing.T, *database){ + "pending": phase("pending"), + "provisioning": phase("provisioning"), + "interrupted": phase("interrupted"), + "compute-ready": phase("compute-ready"), + "unknown-phase": phase("destroyed"), + "claimed": func(_ *testing.T, db *database) { db.item["Owner"] = str("another-runner") }, + "address-kept": node(func(n map[string]any) { n["address"].(map[string]any)["phase"] = "releasing" }), + "no-instance": node(func(n map[string]any) { delete(n, "instanceId") }), + "not-ours": func(_ *testing.T, db *database) { db.item["Kind"] = str("ExistingNetwork") }, + "revision": func(_ *testing.T, db *database) { db.item["Revision"] = number(1) }, + "other-network": func(_ *testing.T, db *database) { db.item["Network"] = str("000000000000/us-west-2/devnet-other") }, + "corrupt": func(_ *testing.T, db *database) { db.item["Data"] = str(value(db.item, "Data") + " {}") }, + } + for name, change := range cases { + t.Run(name, func(t *testing.T) { + d, db, p, next, _ := retiredFixture(t) + ctx := context.Background() + change(t, db) + before := clone(db.item) + // Planning never offers the name: at most the plan the journal already holds. + if succeeds, reused, err := d.Succession(ctx, p); (err != nil && !errors.Is(err, ErrNotRetired)) || succeeds != "" || reused { + t.Fatal("unretired operation offered for reuse", err) + } + if _, err := d.Supersede(ctx, next, "runner"); err == nil { + t.Fatal("unretired operation superseded") + } + if len(db.archived) != 0 || db.replaced != 0 || len(db.item) != len(before) || value(db.item, "Data") != value(before, "Data") || value(db.item, "PlanID") != p.ID || value(db.item, "Owner") != value(before, "Owner") { + t.Fatal("refused handover changed the journal") + } + }) + } + t.Run("another-predecessor", func(t *testing.T) { + d, db, p, _, _ := retiredFixture(t) + other, err := p.Successor(strings.Repeat("a", 64)) + if err != nil { + t.Fatal(err) + } + if _, err = d.Supersede(context.Background(), other, "runner"); err == nil || len(db.archived) != 0 || value(db.item, "PlanID") != p.ID { + t.Fatal("operation superseded by a plan that does not name it", err) + } + }) + t.Run("free-name", func(t *testing.T) { + d, db, p := setup(t) + ctx := context.Background() + if succeeds, reused, err := d.Succession(ctx, p); err != nil || succeeds != "" || reused { + t.Fatal("free name has a succession", err) + } + next, err := p.Successor(strings.Repeat("a", 64)) + if err != nil { + t.Fatal(err) + } + for _, acquire := range []func(context.Context, provision.Plan, string) (provision.Record, error){d.Supersede, d.Acquire} { + if _, err = acquire(ctx, next, "runner"); err == nil || db.item != nil || len(db.archived) != 0 { + t.Fatal("successor plan created an operation with nothing to succeed", err) + } + } + // A live first operation plans as itself, and a changed network is told so. + if _, err = d.Acquire(ctx, p, "runner"); err != nil { + t.Fatal(err) + } + if succeeds, reused, err := d.Succession(ctx, p); err != nil || succeeds != "" || reused { + t.Fatal("live operation cannot be planned again", err) + } + if _, _, err = d.Succession(ctx, next); err == nil { + t.Fatal("succession computed from a successor plan") + } + n := p.Network + n.Chain.Generation++ + changed, err := provision.Prepare(ctx, n, testutil.Identity{Account: n.AWS.AccountID}, &testutil.Cloud{Network: n}) + if err != nil || changed.ID == p.ID { + t.Fatal(err) + } + if succeeds, reused, err := d.Succession(ctx, changed); !errors.Is(err, ErrNotRetired) || succeeds != "" || reused { + t.Fatal("a changed network was offered a live network's name", err) + } + }) +} + +func TestConcurrentSupersedeHasExactlyOneWinner(t *testing.T) { + d, db, p, next, c := retiredFixture(t) + ctx := context.Background() + // A second, different successor of the same retired plan competes too. + n := p.Network + n.Chain.Generation++ + base, err := provision.Prepare(ctx, n, testutil.Identity{Account: n.AWS.AccountID}, c) + if err != nil { + t.Fatal(err) + } + rival, err := base.Successor(p.ID) + if err != nil || rival.ID == next.ID || rival.Key() != next.Key() { + t.Fatal("rival successor invalid", err) + } + type result struct { + plan provision.Plan + err error + } + results := make(chan result, 4) + var wg sync.WaitGroup + for j, plan := range []provision.Plan{next, next, rival, rival} { + owner := fmt.Sprint("runner-", j) + wg.Go(func() { + _, err := d.Supersede(ctx, plan, owner) + results <- result{plan, err} + }) + } + wg.Wait() + close(results) + var winner provision.Plan + wins := 0 + for r := range results { + if r.err == nil { + winner = r.plan + wins++ + } + } + if wins != 1 || db.replaced != 1 || value(db.item, "PlanID") != winner.ID || value(db.item, "Owner") == "" || len(db.archived) != 1 || value(db.archived[p.ArchiveKey()], "PlanID") != p.ID { + t.Fatal("handover was not exclusive", wins, db.replaced) + } + // The loser's plan stays refused: the name now belongs to the winner. + loser := rival + if winner.ID == rival.ID { + loser = next + } + if err = d.Release(ctx, winner, value(db.item, "Owner")); err != nil { + t.Fatal(err) + } + if _, err = d.Supersede(ctx, loser, "late"); err == nil || value(db.item, "PlanID") != winner.ID || value(db.item, "Owner") != "" { + t.Fatal("losing plan replaced the winner", err) + } +} + +func TestHandoverSurvivesLostResponsesAndRepeats(t *testing.T) { + t.Run("stopped-after-archive", func(t *testing.T) { + d, db, p, next, _ := retiredFixture(t) + ctx := context.Background() + before := value(db.item, "Data") + db.replace = "refuse" + if _, err := d.Supersede(ctx, next, "first"); err == nil || db.replaced != 0 || value(db.item, "PlanID") != p.ID || value(db.item, "Owner") != "" || len(db.archived) != 1 { + t.Fatal("interrupted handover left an unexpected journal", err) + } + // Still retired, still plannable, and the archive is not written twice. + if succeeds, reused, err := d.Succession(ctx, p); err != nil || succeeds != p.ID || !reused { + t.Fatal("retired operation lost after an interrupted handover", err) + } + db.replace = "" + if _, err := d.Supersede(ctx, next, "second"); err != nil || db.replaced != 1 || len(db.archived) != 1 || value(db.archived[p.ArchiveKey()], "Data") != before || value(db.item, "Owner") != "second" { + t.Fatal("repeat after an interrupted handover failed", err) + } + }) + t.Run("archive-holds-something-else", func(t *testing.T) { + d, db, p, next, _ := retiredFixture(t) + db.archived = map[string]map[string]types.AttributeValue{p.ArchiveKey(): {"Network": str(p.ArchiveKey()), "PlanID": str(p.ID), "Data": str("{}"), "Revision": number(1)}} + if _, err := d.Supersede(context.Background(), next, "runner"); err == nil || db.replaced != 0 || value(db.item, "PlanID") != p.ID || value(db.archived[p.ArchiveKey()], "Data") != "{}" { + t.Fatal("handover overwrote or ignored different archived evidence", err) + } + }) + t.Run("response-lost", func(t *testing.T) { + d, db, p, next, _ := retiredFixture(t) + ctx := context.Background() + db.replace = "lose" + if _, err := d.Supersede(ctx, next, "lost"); err == nil || db.replaced != 1 { + t.Fatal("expected an ambiguous handover", err) + } + // Like a lost Acquire response: the claim exists and is never stolen. + if _, err := d.Supersede(ctx, next, "new-runner"); err == nil || value(db.item, "Owner") != "lost" { + t.Fatal("ambiguous handover claim was stolen", err) + } + current, owner, err := d.Read(ctx, next) + if err != nil || owner != "lost" || current.Phase != "pending" || current.Predecessor.PlanID != p.ID { + t.Fatal("ambiguous handover is not inspectable", err) + } + // The operator unlocks the stopped runner; the same plan then resumes. + if err = d.Release(ctx, next, "lost"); err != nil { + t.Fatal(err) + } + if _, err = d.Supersede(ctx, next, "new-runner"); err != nil || db.replaced != 1 || len(db.archived) != 1 { + t.Fatal("handover did not resume after unlock", err) + } + }) + t.Run("own-write-retried", func(t *testing.T) { + d, db, p, next, _ := retiredFixture(t) + db.replace = "retry" + r, err := d.Supersede(context.Background(), next, "runner") + if err != nil || r.Predecessor == nil || r.Predecessor.PlanID != p.ID || db.replaced != 1 || value(db.item, "Owner") != "runner" || value(db.item, "PlanID") != next.ID { + t.Fatal("a retry of the accepted handover was treated as a conflict", err) + } + }) +} + +func TestSuccessorIsItselfRetiredAndSucceeded(t *testing.T) { + d, db, p, next, _ := retiredFixture(t) + ctx := context.Background() + r, err := d.Supersede(ctx, next, "runner") + if err != nil { + t.Fatal(err) + } + // The second fleet is provisioned, destroyed and retired in its turn. + for j, target := range next.Targets { + at := r.CreatedAt + r.Nodes[target.Name] = provision.NodeProgress{Phase: "present", AttemptedAt: at, ObservedAt: at, EC2State: "terminated", InstanceID: fmt.Sprintf("i-%017x", 100+j), Address: &provision.AddressProgress{Phase: "released", AttemptedAt: at, AllocationID: fmt.Sprintf("eipalloc-%017x", 100+j), PublicIP: fmt.Sprintf("198.51.100.%d", 100+j)}} + } + r.Phase = "addresses-released" + r.Revision++ + if err = d.Save(ctx, r, "runner"); err != nil { + t.Fatal(err) + } + if err = d.Release(ctx, next, "runner"); err != nil { + t.Fatal(err) + } + succeeds, reused, err := d.Succession(ctx, p) + if err != nil || succeeds != next.ID || !reused { + t.Fatal("retired successor not recognised", err) + } + third, err := p.Successor(succeeds) + if err != nil || third.ID == next.ID || third.ID == p.ID { + t.Fatal("second successor repeats an earlier plan", err) + } + fresh, err := d.Supersede(ctx, third, "runner") + if err != nil || fresh.Predecessor.PlanID != next.ID || len(fresh.Predecessor.Instances) != len(next.Targets) || value(db.item, "PlanID") != third.ID { + t.Fatal("second handover failed", err) + } + // Both earlier operations are kept, each under its own archive key. + if len(db.archived) != 2 || value(db.archived[p.ArchiveKey()], "PlanID") != p.ID || value(db.archived[next.ArchiveKey()], "PlanID") != next.ID { + t.Fatal("an archived operation was lost or replaced") + } + for _, old := range []provision.Plan{p, next} { + if archived, err := d.Archived(ctx, old); err != nil || archived.Plan.ID != old.ID { + t.Fatal("archived operation unreadable", err) + } + if _, err := d.Supersede(ctx, old, "late"); err == nil { + t.Fatal("an archived plan took the name back") + } + } +} From 50db4be1f4ecbb1eace339ed8c9cef786a037b38 Mon Sep 17 00:00:00 2001 From: ktechmidas <9920871+ktechmidas@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:44:55 +0300 Subject: [PATCH 3/4] feat(cli): provision-plan succeeds a retired plan; document name reuse provision-plan reads the journal (read-only) and, when the network's key holds a retired operation, writes the plan that succeeds it and says so. operation shows a plan's archived record once its name has been reused. docs/provisioning.md states the exact condition under which a name is released, the archive and handover steps, the live-scope exception, and what reuse does not change: the chain identity follows the name and generation, so a name reused at the same generation is the same Core chain name and Platform chain ID on a new genesis. Co-Authored-By: Claude Fable 5.2 --- docs/ipam.md | 2 + docs/provisioning.md | 95 ++++++++++++++++++++++++++++++++++++--- internal/cli/provision.go | 30 ++++++++++++- 3 files changed, 119 insertions(+), 8 deletions(-) diff --git a/docs/ipam.md b/docs/ipam.md index 658ad72..760b585 100644 --- a/docs/ipam.md +++ b/docs/ipam.md @@ -92,6 +92,8 @@ tag search or a failed read. This cannot hide a retained address whose tags were changed. Successful cleanup marks the allocation retired; provisioning cannot resurrect it. Partial cleanup remains resumable. Root disks and other resources need their own reviewed cleanup. +A retired allocation is also what releases the network's name: a new +`provision-plan` for it then [succeeds the retired plan](provisioning.md#reusing-the-name-of-a-destroyed-network). ## IAM and proof boundary diff --git a/docs/provisioning.md b/docs/provisioning.md index f965000..52eeb0b 100644 --- a/docs/provisioning.md +++ b/docs/provisioning.md @@ -70,11 +70,13 @@ roles table deletion or state item deletion permissions. Restrict roles to the canonical table: choosing another table is not a recovery procedure. The network key is `account/region/name`, intentionally **without generation**. -A different generation or plan cannot bypass a previous claim/operation. This -milestone stores one immutable provision operation per network, not a general -workflow history service. It refuses changed plans even after completion. Reset, -replacement, and state migration need their own reviewed lifecycle implementation; -never delete the record just to make a retry pass. +A different generation or plan cannot bypass a previous claim/operation. The +journal holds one immutable provision operation per network key, not a general +workflow history service, and refuses changed plans even after completion. A key +changes hands in exactly one reviewed way: +[the name of a destroyed network is reused](#reusing-the-name-of-a-destroyed-network). +Reset, in-place replacement and state migration are not implemented; never delete +or edit a record to make a retry pass. ## Human and agent command path @@ -156,6 +158,87 @@ stale runner from changing state, **not** an already-issued EC2 API request from completing. There is no claim that DynamoDB fencing fences EC2 itself. The stopped runner prerequisite matters. Never unlock just because a job appears slow. +## Reusing the name of a destroyed network + +A network name is released for reuse when, and only when, the operation the +journal holds under its key proves that network is gone: + +- its phase is `addresses-released`, which only `dashnet release-addresses` + writes, after it has seen **every** journaled instance terminated under this + plan's ownership tags and client token (or reported absent by its exact ID once + EC2 purged it) and **every** address absent by its exact allocation ID; +- every target has a journaled instance ID, and no address is in any state but + `released`; +- the record is consistent with its own plan, schema and revision; +- no runner holds its claim. + +Nothing weaker counts. A terminated fleet whose addresses were not released, an +interrupted or pending operation, and a claimed record all keep their name. A +plan without IPAM public addresses (`publicIpv4: false`) has no address +retirement and therefore **no proof**: its name is never released by this tool. + +Reuse is part of the plan, so it is reviewed like any other plan: + +```sh +# After release-addresses for the old plan. Reads the journal (read-only). +dashnet provision-plan --network networks/devnet-lab.yaml \ + --profile YOUR_PROFILE --out out/lab-ec2-plan-2.json +``` + +When the key holds a retired operation, `provision-plan` says so and writes a +plan with `"succeeds": "RETIRED_PLAN_ID"`. That field is covered by the plan ID, +so the new plan differs from the retired one even if the network definition is +unchanged: every EC2 client token and every `dashnet:plan` tag of the new fleet +is new. `provision --confirm NEW_PLAN_ID` then, and only then: + +1. reads the live record and checks again that it is exactly the plan named by + `succeeds`, retired and unclaimed; +2. copies it verbatim to the archive key `account/region/name#retired#OLD_PLAN_ID` + in the same table (create-if-absent; a repeat finds the same item and goes on, + a different item there stops the handover); +3. replaces the live item with the new plan's fresh operation, claimed by this + runner, in one write conditional on the old plan ID, the revision that was + read and the absence of a claim. + +Nothing is deleted. Of two runners, one conditional write wins; the other finds +an operation that is no longer the one it read and is refused by the ordinary +plan and claim rules. If the response to step 3 is lost, the claim may exist: +inspect and unlock it exactly as after any [crashed runner](#crashed-runner--persistent-claim), +then rerun the same plan. An SDK retry of the accepted write is recognised as +this runner's own. `bootstrap`, `deploy`, `join`, `upgrade` and +`release-addresses` never hand a name over; only `provision` does. + +The successor starts empty. No instance, address, bootstrap, deployment, runtime +or upgrade state is carried over, and nothing of the old fleet is adopted. Its +record keeps only a `predecessor` entry: the retired plan ID, the archive key and +the retired instance IDs. `dashnet operation --plan OLD_PLAN` keeps working and +shows the archived record with `"archived": true`. Planning the same network +definition again returns the same successor plan, so a lost plan file does not +strand an interrupted operation. + +The live scope is checked as strictly as for a first provision: any instance +carrying the network tag that is not this plan's own blocks it. The single +exception is the retired predecessor's own instances, which EC2 keeps listing as +`terminated` for about an hour: an instance is skipped only if it is +`terminated`, carries the predecessor's `dashnet:plan` tag **and** its ID is one +the predecessor journaled. Such an instance can never run again. A stopped, +shutting-down or running one, or a terminated one the predecessor did not +record, still blocks. Only the direct predecessor is known, so a third reuse +within that hour waits for EC2 to purge the first fleet's instances. + +What reuse does **not** change or cover: + +- **Chain identity.** The Core devnet name and the Platform chain ID follow the + network name and generation. A name reused at the same generation is the same + chain identity as the destroyed network, on a new genesis. Raise the + generation in the network definition if the two must be distinguishable. +- **Root volumes.** They are retained on termination and keep the old plan's + tags. They are never adopted and never checked; delete them separately. +- **Existing-network enrollments.** A name enrolled with the managed-network + commands shares the table and key space and is never retired. +- **Old binaries.** A release without this feature rejects a plan with + `succeeds` and a record with `predecessor` as unknown fields. + ## Permissions and Actions boundary The CLI uses the normal AWS SDK credential chain. `--profile` is for terminals; @@ -169,7 +252,7 @@ Minimum API surfaces to scope in IAM are: | Instance creation | `ec2:RunInstances`, `ec2:CreateTags` for tag-on-create; scoped AMI, subnet, groups, key pair, and instance/volume resources | | IPAM public addresses | `ec2:DescribeIpamPools`, `DescribeAddresses`, `AllocateAddress`, `AssociateAddress`, scoped `CreateTags`; see [IPAM](ipam.md) | | Explicit retired-address cleanup | `ec2:ReleaseAddress` after every original instance is terminated and every owned address is detached | -| Journal | `dynamodb:DescribeTable`, `GetItem`, `PutItem`, `UpdateItem` on the one regional table | +| Journal | `dynamodb:DescribeTable`, `GetItem`, `PutItem`, `UpdateItem` on the one regional table. Name reuse needs nothing more: the archive and the handover are conditional `PutItem`s. `provision-plan` now also reads the network's item (`GetItem`) | Encrypted volumes using account-specific KMS policies may require additional permissions. IAM restrictions are defense in depth; these action names alone are diff --git a/internal/cli/provision.go b/internal/cli/provision.go index 7b97e45..4f4d993 100644 --- a/internal/cli/provision.go +++ b/internal/cli/provision.go @@ -117,18 +117,44 @@ func runProvision(ctx context.Context, args []string, out, stderr io.Writer, ver } switch command { case "provision-plan": + // A destroyed network's name is reused by the plan that succeeds its + // retired operation. Read-only here: provision decides again. + succeeds, reused, err := store.Succession(ctx, p) + switch { + case errors.Is(err, journal.ErrNotRetired): + fmt.Fprintln(stderr, "note: this name already has an operation for a different plan, and it is not retired;", err) + fmt.Fprintln(stderr, "note: provision will refuse this plan") + case err != nil: + return fmt.Errorf("read journal for name reuse: %w", err) + case succeeds != "": + if p, err = p.Successor(succeeds); err != nil { + return err + } + if reused { + fmt.Fprintln(stderr, "This name belonged to a destroyed network, retired plan "+succeeds+".") + fmt.Fprintln(stderr, "This plan succeeds it: provision archives that operation and starts a new one. The chain identity follows the name and generation, so it is unchanged.") + } + } fmt.Fprintln(stderr, p.Footprint()) fmt.Fprintln(stderr, "Read-only footprint check, not a cost quote or proof of SSH/OS/chain readiness. Review networking and AMI contents separately.") return emit(out, output, p) case "operation": r, owner, err := store.Read(ctx, p) + archived := false if err != nil { - return err + // The name may have been reused since: the operation is then kept + // under its archive key, unclaimed and read-only. + kept, e := store.Archived(ctx, p) + if e != nil { + return err + } + r, owner, archived = kept, "", true } return emit(out, output, struct { Owner string `json:"owner"` + Archived bool `json:"archived,omitempty"` Operation provision.Record `json:"operation"` - }{owner, r}) + }{owner, archived, r}) case "operation-unlock": if _, owner, err := store.Read(ctx, p); err != nil { return err From 311fd83d77897c956f337459dad2821ab66f9541 Mon Sep 17 00:00:00 2001 From: ktechmidas <9920871+ktechmidas@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:10:25 +0300 Subject: [PATCH 4/4] fix(journal): a stopped handover can no longer block a name for good Review found a sequence that locked a released name permanently: 1. the handover archived the retired operation, then stopped before the replacement; 2. a retried deployment of the old network failed, but had claimed the retired operation and written two checkpoints into it (revision N to N+2); 3. every later handover found an archive that no longer matched the live item and refused with "the archive key holds a different operation". Two causes, both fixed. A retired operation could still be claimed and changed. deploy never looked at the compute phase, and nothing in the store stopped it. Acquire now refuses an operation in `addresses-released` with provision.ErrRetired, for every stage, before claiming it; if it is retired between the read and the claim, the claim is given back unused. Address cleanup keeps its own store, can still be repeated, and writes nothing to a retired operation. The archive key did not say which state it held, so a copy of an earlier revision stood in for, and then contradicted, the item being replaced. The key is now `#retired##`. A handover archives exactly the revision it read, a repeat of the same revision finds its own copy, and a copy of any other revision is neither compared nor touched. The successor records the revision and key it replaced, and Archived follows that chain. The two writes stay two writes. A transaction was considered and left out: it would need a client call this package does not use, and its permission requirements could not be checked here. Co-Authored-By: Claude Fable 5.2 --- docs/provisioning.md | 25 ++- internal/journal/dynamodb.go | 108 ++++++++---- .../journal/supersede_integration_test.go | 32 +++- internal/journal/supersede_test.go | 155 ++++++++++++++++-- internal/provision/plan.go | 12 +- internal/provision/reuse_test.go | 11 +- internal/provision/runner.go | 12 +- 7 files changed, 290 insertions(+), 65 deletions(-) diff --git a/docs/provisioning.md b/docs/provisioning.md index 52eeb0b..01633b4 100644 --- a/docs/provisioning.md +++ b/docs/provisioning.md @@ -193,12 +193,13 @@ is new. `provision --confirm NEW_PLAN_ID` then, and only then: 1. reads the live record and checks again that it is exactly the plan named by `succeeds`, retired and unclaimed; -2. copies it verbatim to the archive key `account/region/name#retired#OLD_PLAN_ID` - in the same table (create-if-absent; a repeat finds the same item and goes on, - a different item there stops the handover); +2. copies it verbatim to the archive key + `account/region/name#retired#OLD_PLAN_ID#REVISION` in the same table, where + `REVISION` is the journal revision that was read (create-if-absent; a repeat + finds the same item and goes on); 3. replaces the live item with the new plan's fresh operation, claimed by this - runner, in one write conditional on the old plan ID, the revision that was - read and the absence of a claim. + runner, in one write conditional on the old plan ID, that same revision and + the absence of a claim. Nothing is deleted. Of two runners, one conditional write wins; the other finds an operation that is no longer the one it read and is refused by the ordinary @@ -208,11 +209,23 @@ then rerun the same plan. An SDK retry of the accepted write is recognised as this runner's own. `bootstrap`, `deploy`, `join`, `upgrade` and `release-addresses` never hand a name over; only `provision` does. +Steps 2 and 3 are two writes, not one transaction. A handover that stops between +them leaves the retired operation in place with a copy beside it, and the next +`provision` of the same plan goes on from there. Because the archive key names +the revision, a copy can never block a later attempt: if the retired operation +has a different revision by then, that revision is archived under its own key and +the earlier copy stays where it is. + +A retired operation does not change any more. Once an operation is in +`addresses-released`, `provision`, `bootstrap`, `deploy`, `join` and `upgrade` +refuse to claim it ("allocation retired"), so none of them can write a checkpoint +into it. `release-addresses` can still be repeated against it and changes nothing. + The successor starts empty. No instance, address, bootstrap, deployment, runtime or upgrade state is carried over, and nothing of the old fleet is adopted. Its record keeps only a `predecessor` entry: the retired plan ID, the archive key and the retired instance IDs. `dashnet operation --plan OLD_PLAN` keeps working and -shows the archived record with `"archived": true`. Planning the same network +shows the archived record, the revision that was replaced, with `"archived": true`. Planning the same network definition again returns the same successor plan, so a lost plan file does not strand an interrupted operation. diff --git a/internal/journal/dynamodb.go b/internal/journal/dynamodb.go index e546e59..dba72d5 100644 --- a/internal/journal/dynamodb.go +++ b/internal/journal/dynamodb.go @@ -191,10 +191,17 @@ func (d Dynamo) Succession(ctx context.Context, p provision.Plan) (succeeds stri // Supersede is Acquire for a plan that succeeds a retired one. The key is // handed over only if the live item is exactly the plan p names, unclaimed and // retired. Nothing is deleted: the retired item is first copied, verbatim, to -// its archive key (create-if-absent, so a repeat is harmless), then replaced -// by p's fresh operation in one write conditional on the plan, the revision -// that was read and the absence of a claim. Of two runners, one write wins and -// the other finds an operation that is no longer the one it read. +// the archive key of the revision that was read (create-if-absent, so a repeat +// is harmless), then replaced by p's fresh operation in one write conditional +// on the plan, that same revision and the absence of a claim. Of two runners, +// one write wins and the other finds an operation that is no longer the one +// it read. +// +// The two writes are not atomic. If the handover stops between them, the +// retired operation simply has a copy, and the next attempt goes on. If the +// retired operation has changed by then (an older release could still write +// to one), that attempt archives the new revision under its own key: a copy +// of an earlier revision never stands in its way. func (d Dynamo) Supersede(ctx context.Context, p provision.Plan, owner string) (provision.Record, error) { if owner == "" { return provision.Record{}, errors.New("empty runner ID") @@ -228,14 +235,14 @@ func (d Dynamo) Supersede(ctx context.Context, p provision.Plan, owner string) ( if err != nil { return provision.Record{}, err } - archive := map[string]types.AttributeValue{"Network": str(old.Plan.ArchiveKey()), "PlanID": item["PlanID"], "Data": item["Data"], "Revision": item["Revision"]} + archive := map[string]types.AttributeValue{"Network": str(old.Plan.ArchiveKey(old.Revision)), "PlanID": item["PlanID"], "Data": item["Data"], "Revision": item["Revision"]} _, err = d.Client.PutItem(ctx, &dynamodb.PutItemInput{TableName: aws.String(d.Table), Item: archive, ConditionExpression: aws.String("attribute_not_exists(#network)"), ExpressionAttributeNames: map[string]string{"#network": "Network"}}) if conditional(err) { - // An earlier attempt archived it. It must be this very operation. + // An earlier attempt archived this revision. It must be this very item. var kept *dynamodb.GetItemOutput kept, err = d.Client.GetItem(ctx, &dynamodb.GetItemInput{TableName: aws.String(d.Table), Key: map[string]types.AttributeValue{"Network": archive["Network"]}, ConsistentRead: aws.Bool(true)}) if err == nil && (kept == nil || len(kept.Item) != len(archive) || value(kept.Item, "PlanID") != value(archive, "PlanID") || value(kept.Item, "Data") != value(archive, "Data") || !sameNumber(kept.Item["Revision"], archive["Revision"])) { - err = errors.New("the archive key holds a different operation; nothing replaced") + err = errors.New("the archive key of this revision holds different data; nothing replaced") } } if err != nil { @@ -271,26 +278,46 @@ func (d Dynamo) Supersede(ctx context.Context, p provision.Plan, owner string) ( return d.Acquire(ctx, p, owner) } -// Archived reads the operation of a plan whose network key has been reused. +// Archived reads the operation of a plan whose network key has been reused. It +// is found from the live operation, each operation naming the archive key of +// the one it replaced. func (d Dynamo) Archived(ctx context.Context, p provision.Plan) (provision.Record, error) { - out, err := d.Client.GetItem(ctx, &dynamodb.GetItemInput{TableName: aws.String(d.Table), Key: map[string]types.AttributeValue{"Network": str(p.ArchiveKey())}, ConsistentRead: aws.Bool(true)}) + item, err := d.readItem(ctx, p) if err != nil { return provision.Record{}, err } - if out == nil || len(out.Item) == 0 { - return provision.Record{}, ErrNotFound - } - if len(out.Item) != 4 || value(out.Item, "Network") != p.ArchiveKey() || value(out.Item, "PlanID") != p.ID { - return provision.Record{}, errors.New("invalid archived operation") - } - r, err := decodePayload(out.Item) - if err != nil { - return r, err - } - if err = r.ValidateAddressCleanup(p); err != nil { - return r, err + for range 100 { + newer, err := decodePayload(item) + if err != nil { + return provision.Record{}, err + } + if newer.Predecessor == nil { + return provision.Record{}, ErrNotFound + } + out, err := d.Client.GetItem(ctx, &dynamodb.GetItemInput{TableName: aws.String(d.Table), Key: map[string]types.AttributeValue{"Network": str(newer.Predecessor.Archive)}, ConsistentRead: aws.Bool(true)}) + if err != nil { + return provision.Record{}, err + } + if out == nil || len(out.Item) != 4 || value(out.Item, "Network") != newer.Predecessor.Archive || value(out.Item, "PlanID") != newer.Predecessor.PlanID { + return provision.Record{}, errors.New("missing or invalid archived operation") + } + item = out.Item + if newer.Predecessor.PlanID != p.ID { + continue + } + r, err := decodePayload(item) + if err != nil { + return r, err + } + if r.Revision != newer.Predecessor.Revision { + return r, errors.New("archived operation is not the revision that was replaced") + } + if err = r.ValidateAddressCleanup(p); err != nil { + return r, err + } + return r, r.Retired() } - return r, r.Retired() + return provision.Record{}, ErrNotFound } func sameNumber(a, b types.AttributeValue) bool { @@ -327,14 +354,7 @@ func (d Dynamo) Acquire(ctx context.Context, p provision.Plan, owner string) (pr if p.Succeeds != "" { // A successor's operation is created only by Supersede, from provision. // Every other stage may claim it once it exists, and nothing else. - if _, _, err := d.Read(ctx, p); err != nil { - return provision.Record{}, err - } - claimed, err := d.claim(ctx, p, owner) - if err != nil { - return provision.Record{}, err - } - return decode(claimed, p) + return d.claimLive(ctx, p, owner) } r := provision.NewRecord(p) data, err := encode(r) @@ -354,14 +374,34 @@ func (d Dynamo) Acquire(ctx context.Context, p provision.Plan, owner string) (pr return r, err } // Do not overwrite an existing record. It may be an interrupted operation. - if _, _, err = d.Read(ctx, p); err != nil { - return r, err + return d.claimLive(ctx, p, owner) +} + +// claimLive claims an existing operation for a stage that will work on it. A +// retired operation is never claimed here: it would let that stage write new +// checkpoints into the proof that its network is gone. Only address cleanup, +// through its own store, still reads one, and it changes nothing. +func (d Dynamo) claimLive(ctx context.Context, p provision.Plan, owner string) (provision.Record, error) { + existing, _, err := d.Read(ctx, p) + if err != nil { + return provision.Record{}, err + } + if existing.Phase == "addresses-released" { + return provision.Record{}, provision.ErrRetired } claimed, err := d.claim(ctx, p, owner) if err != nil { - return r, err + return provision.Record{}, err + } + r, err := decode(claimed, p) + if err == nil && r.Phase == "addresses-released" { + // Retired between the read and the claim: give the claim back unused. + if err = d.Release(ctx, p, owner); err != nil { + return provision.Record{}, errors.Join(provision.ErrRetired, err) + } + return provision.Record{}, provision.ErrRetired } - return decode(claimed, p) + return r, err } // claim is the same non-expiring conditional gate for active execution and diff --git a/internal/journal/supersede_integration_test.go b/internal/journal/supersede_integration_test.go index 8817887..7ac313c 100644 --- a/internal/journal/supersede_integration_test.go +++ b/internal/journal/supersede_integration_test.go @@ -3,6 +3,7 @@ package journal import ( "context" "encoding/json" + "errors" "fmt" "os" "sync" @@ -32,6 +33,7 @@ func TestJournalDynamoLocalNameReuse(t *testing.T) { defer cancel() _, fixture, p, next, cloud := retiredFixture(t) retiredItem := clone(fixture.item) + archive := archiveKey(t, p, retiredItem) table := fmt.Sprintf("journal-test-%d", time.Now().UnixNano()) client := dynamodb.NewFromConfig(aws.Config{ Region: p.Network.AWS.Region, @@ -82,7 +84,7 @@ func TestJournalDynamoLocalNameReuse(t *testing.T) { } unchanged := func(want map[string]types.AttributeValue) bool { got := get(p.Key()) - return len(got) == len(want) && value(got, "PlanID") == value(want, "PlanID") && value(got, "Data") == value(want, "Data") && value(got, "Owner") == value(want, "Owner") && sameNumber(got["Revision"], want["Revision"]) && len(get(p.ArchiveKey())) == 0 + return len(got) == len(want) && value(got, "PlanID") == value(want, "PlanID") && value(got, "Data") == value(want, "Data") && value(got, "Owner") == value(want, "Owner") && sameNumber(got["Revision"], want["Revision"]) && len(get(archive)) == 0 } t.Run("refused while claimed or not retired", func(t *testing.T) { @@ -110,12 +112,31 @@ func TestJournalDynamoLocalNameReuse(t *testing.T) { if _, err = store.Supersede(ctx, next, "runner"); err == nil || !unchanged(running) { t.Fatal("live operation superseded", err) } + // A retired operation that today's rules accept in full is still never claimed. + for _, field := range historyFields { + delete(fields, field) + } + fields["phase"] = json.RawMessage(`"addresses-released"`) + if data, err = json.Marshal(fields); err != nil { + t.Fatal(err) + } + closed := clone(retiredItem) + closed["Data"] = str(string(data)) + put(closed) + if _, err = store.Acquire(ctx, p, "deploy"); !errors.Is(err, provision.ErrRetired) || !unchanged(closed) { + t.Fatal("a stage claimed a retired operation", err) + } put(retiredItem) if _, err = store.Acquire(ctx, next, "bootstrap"); err == nil || !unchanged(retiredItem) { t.Fatal("a stage other than provision took the name over", err) } }) + // A copy of an earlier revision, left by a handover that stopped half way, + // before the retired operation changed. It must neither block nor be touched. + stale := map[string]types.AttributeValue{"Network": str(p.ArchiveKey(1)), "PlanID": str(p.ID), "Data": str("{}"), "Revision": number(1)} + put(stale) + n := p.Network n.Chain.Generation++ base, err := provision.Prepare(ctx, n, testutil.Identity{Account: n.AWS.AccountID}, cloud) @@ -161,7 +182,7 @@ func TestJournalDynamoLocalNameReuse(t *testing.T) { if wins != 1 { t.Fatal("handover was not exclusive", wins) } - live, kept := get(p.Key()), get(p.ArchiveKey()) + live, kept := get(p.Key()), get(archive) if value(live, "PlanID") != winner.ID || value(live, "Owner") != owner || !sameNumber(live["Revision"], number(0)) { t.Fatal("live item is not the winner's claimed operation") } @@ -172,9 +193,12 @@ func TestJournalDynamoLocalNameReuse(t *testing.T) { if err != nil || archived.Phase != "addresses-released" || archived.Plan.ID != p.ID { t.Fatal("archived operation unreadable", err) } - if record.Predecessor == nil || record.Predecessor.PlanID != p.ID || record.Predecessor.Archive != p.ArchiveKey() { + if record.Predecessor == nil || record.Predecessor.PlanID != p.ID || record.Predecessor.Archive != archive { t.Fatal("successor lost its predecessor") } + if earlier := get(p.ArchiveKey(1)); len(earlier) != 4 || value(earlier, "Data") != "{}" { + t.Fatal("an archive of an earlier revision was touched") + } }) t.Run("successor is an ordinary operation and the loser stays refused", func(t *testing.T) { @@ -202,7 +226,7 @@ func TestJournalDynamoLocalNameReuse(t *testing.T) { t.Fatal(err) } } - if value(get(p.ArchiveKey()), "Data") != value(retiredItem, "Data") { + if value(get(archive), "Data") != value(retiredItem, "Data") { t.Fatal("archive changed after the handover") } if _, _, err := store.Read(ctx, p); err == nil { diff --git a/internal/journal/supersede_test.go b/internal/journal/supersede_test.go index 6fcc6f5..bbdcc54 100644 --- a/internal/journal/supersede_test.go +++ b/internal/journal/supersede_test.go @@ -5,10 +5,12 @@ import ( "encoding/json" "errors" "fmt" + "strconv" "strings" "sync" "testing" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/dashpay/dash-network-go/internal/provision" "github.com/dashpay/dash-network-go/internal/testutil" @@ -40,6 +42,21 @@ func retiredFixture(t *testing.T) (Dynamo, *database, provision.Plan, provision. return d, db, p, next, c } +// archiveKey is where a plan's operation is kept once the given item, its live +// record, has been replaced. +func archiveKey(t *testing.T, p provision.Plan, item map[string]types.AttributeValue) string { + t.Helper() + n, ok := item["Revision"].(*types.AttributeValueMemberN) + if !ok { + t.Fatal("journal item has no revision") + } + revision, err := strconv.ParseInt(n.Value, 10, 64) + if err != nil { + t.Fatal(err) + } + return p.ArchiveKey(revision) +} + // rewrite changes the stored payload of the live item, as raw JSON. func rewrite(t *testing.T, db *database, change func(map[string]json.RawMessage)) { t.Helper() @@ -70,15 +87,16 @@ func TestRetiredNameIsArchivedAndHandedToItsSuccessor(t *testing.T) { if err != nil || r.Phase != "pending" || r.Revision != 0 || r.Plan.ID != next.ID || r.Predecessor == nil || r.Predecessor.PlanID != p.ID || len(r.Predecessor.Instances) != len(p.Targets) || r.Bootstrap != nil || r.Deployment != nil || r.Runtime != nil || r.Upgrade != nil { t.Fatal("successor did not start a fresh operation", err) } - kept := db.archived[p.ArchiveKey()] - if len(db.archived) != 1 || len(kept) != 4 || value(kept, "Network") != p.ArchiveKey() || value(kept, "PlanID") != p.ID || value(kept, "Data") != value(before, "Data") || !sameNumber(kept["Revision"], before["Revision"]) { + archive := archiveKey(t, p, before) + kept := db.archived[archive] + if len(db.archived) != 1 || len(kept) != 4 || r.Predecessor.Archive != archive || value(kept, "Network") != archive || value(kept, "PlanID") != p.ID || value(kept, "Data") != value(before, "Data") || !sameNumber(kept["Revision"], before["Revision"]) { t.Fatal("retired operation was not archived verbatim") } if value(db.item, "Network") != p.Key() || value(db.item, "PlanID") != next.ID || value(db.item, "Owner") != "one" || !sameNumber(db.item["Revision"], number(0)) { t.Fatal("live item is not the successor's claimed operation") } current, owner, err := d.Read(ctx, next) - if err != nil || owner != "one" || current.Predecessor.Archive != p.ArchiveKey() || current.Predecessor.Revision != r.Predecessor.Revision { + if err != nil || owner != "one" || current.Predecessor.Archive != archive || current.Predecessor.Revision != r.Predecessor.Revision { t.Fatal("successor operation unreadable", err) } if _, _, err = d.Read(ctx, p); err == nil { @@ -118,11 +136,121 @@ func TestRetiredNameIsArchivedAndHandedToItsSuccessor(t *testing.T) { if err != nil || succeeds != p.ID || reused { t.Fatal("an interrupted successor cannot be planned again", err) } - if value(db.archived[p.ArchiveKey()], "Data") != value(before, "Data") { + if value(db.archived[archive], "Data") != value(before, "Data") { t.Fatal("archive changed after the handover") } } +// A retired operation is the proof that releases its network's name. No stage +// may claim it and write new checkpoints into it: a retried deployment used to. +func TestRetiredOperationIsNeverClaimedOrChangedAgain(t *testing.T) { + d, db, p, next, _ := retiredFixture(t) + ctx := context.Background() + // A record today's rules accept in full, so only its retirement stands in the way. + rewrite(t, db, func(fields map[string]json.RawMessage) { + for _, field := range historyFields { + delete(fields, field) + } + }) + if r, _, err := d.Read(ctx, p); err != nil || r.Phase != "addresses-released" { + t.Fatal("fixture is not a fully valid retired operation", err) + } + before, saves := clone(db.item), db.saves + untouched := func() bool { + return len(db.item) == len(before) && value(db.item, "Owner") == "" && value(db.item, "Data") == value(before, "Data") && sameNumber(db.item["Revision"], before["Revision"]) && db.saves == saves + } + if _, err := d.Acquire(ctx, p, "deploy"); !errors.Is(err, provision.ErrRetired) || !untouched() { + t.Fatal("a stage claimed a retired operation", err) + } + // Retired between the stage's read and its claim: the claim is given back unused. + retiredData := value(db.item, "Data") + rewrite(t, db, func(fields map[string]json.RawMessage) { fields["phase"] = json.RawMessage(`"compute-ready"`) }) + db.claimResponse = func(out *dynamodb.UpdateItemOutput) (*dynamodb.UpdateItemOutput, error) { + db.item["Data"] = str(retiredData) + out.Attributes["Data"] = str(retiredData) + return out, nil + } + if _, err := d.Acquire(ctx, p, "deploy"); !errors.Is(err, provision.ErrRetired) || !untouched() { + t.Fatal("a stage kept a claim on an operation retired under it", err) + } + db.claimResponse = nil + // Address cleanup may still look, and changes nothing. The name is then handed over. + cleanup := d.AddressCleanup() + if r, err := cleanup.Acquire(ctx, p, "cleanup"); err != nil || r.Phase != "addresses-released" { + t.Fatal("address cleanup can no longer replay a retired operation", err) + } + if err := cleanup.Release(ctx, p, "cleanup"); err != nil || !untouched() { + t.Fatal("address cleanup changed a retired operation", err) + } + r, err := d.Supersede(ctx, next, "runner") + if err != nil || r.Predecessor.Archive != archiveKey(t, p, before) || value(db.archived[r.Predecessor.Archive], "Data") != value(before, "Data") { + t.Fatal("handover failed after refused claims", err) + } + // The successor, once retired in its turn, is as closed as its predecessor. + for _, target := range next.Targets { + at := r.CreatedAt + r.Nodes[target.Name] = provision.NodeProgress{Phase: "present", AttemptedAt: at, ObservedAt: at, EC2State: "terminated", InstanceID: "i-0" + target.Name} + } + r.Phase = "addresses-released" + r.Revision++ + if err = d.Save(ctx, r, "runner"); err != nil { + t.Fatal(err) + } + if err = d.Release(ctx, next, "runner"); err != nil { + t.Fatal(err) + } + for _, acquire := range []func(context.Context, provision.Plan, string) (provision.Record, error){d.Acquire, d.Supersede} { + if _, err = acquire(ctx, next, "late"); !errors.Is(err, provision.ErrRetired) || value(db.item, "Owner") != "" { + t.Fatal("a retired successor was claimed again", err) + } + } +} + +// The sequence that blocked a name for good: the handover archived the retired +// operation and stopped; the operation then changed (two more checkpoints, from +// a release that still let a stage claim it); every later handover found an +// archive that no longer matched and refused. +func TestHandoverIsNotBlockedByAnArchiveOfAnEarlierRevision(t *testing.T) { + d, db, p, next, _ := retiredFixture(t) + ctx := context.Background() + first := archiveKey(t, p, db.item) + original := value(db.item, "Data") + db.replace = "refuse" + if _, err := d.Supersede(ctx, next, "first"); err == nil || len(db.archived) != 1 || value(db.archived[first], "Data") != original || value(db.item, "PlanID") != p.ID { + t.Fatal("handover did not stop after the archive", err) + } + db.replace = "" + var revision int64 + rewrite(t, db, func(fields map[string]json.RawMessage) { + if err := json.Unmarshal(fields["revision"], &revision); err != nil { + t.Fatal(err) + } + revision += 2 + fields["revision"], _ = json.Marshal(revision) + fields["lastError"] = json.RawMessage(`"deployment retried against a retired network"`) + }) + db.item["Revision"] = number(revision) + second, changed := archiveKey(t, p, db.item), value(db.item, "Data") + if second == first || changed == original { + t.Fatal("fixture did not change the retired operation") + } + if succeeds, reused, err := d.Succession(ctx, p); err != nil || succeeds != p.ID || !reused { + t.Fatal("changed retired operation is no longer plannable", err) + } + r, err := d.Supersede(ctx, next, "second") + if err != nil || db.replaced != 1 || value(db.item, "PlanID") != next.ID || value(db.item, "Owner") != "second" || r.Predecessor.Archive != second || r.Predecessor.Revision != revision { + t.Fatal("handover blocked by an archive of an earlier revision", err) + } + // Both copies are kept: the one that was replaced, and the earlier one. + if len(db.archived) != 2 || value(db.archived[second], "Data") != changed || value(db.archived[first], "Data") != original { + t.Fatal("an archived revision was lost or overwritten") + } + archived, err := d.Archived(ctx, p) + if err != nil || archived.Revision != revision || archived.LastError == "" { + t.Fatal("the archived operation is not the revision that was replaced", err) + } +} + func TestOnlyARetiredUnclaimedOperationIsSuperseded(t *testing.T) { phase := func(name string) func(*testing.T, *database) { return func(t *testing.T, db *database) { @@ -239,6 +367,7 @@ func TestConcurrentSupersedeHasExactlyOneWinner(t *testing.T) { err error } results := make(chan result, 4) + archive := archiveKey(t, p, db.item) var wg sync.WaitGroup for j, plan := range []provision.Plan{next, next, rival, rival} { owner := fmt.Sprint("runner-", j) @@ -257,7 +386,7 @@ func TestConcurrentSupersedeHasExactlyOneWinner(t *testing.T) { wins++ } } - if wins != 1 || db.replaced != 1 || value(db.item, "PlanID") != winner.ID || value(db.item, "Owner") == "" || len(db.archived) != 1 || value(db.archived[p.ArchiveKey()], "PlanID") != p.ID { + if wins != 1 || db.replaced != 1 || value(db.item, "PlanID") != winner.ID || value(db.item, "Owner") == "" || len(db.archived) != 1 || value(db.archived[archive], "PlanID") != p.ID { t.Fatal("handover was not exclusive", wins, db.replaced) } // The loser's plan stays refused: the name now belongs to the winner. @@ -277,7 +406,7 @@ func TestHandoverSurvivesLostResponsesAndRepeats(t *testing.T) { t.Run("stopped-after-archive", func(t *testing.T) { d, db, p, next, _ := retiredFixture(t) ctx := context.Background() - before := value(db.item, "Data") + before, archive := value(db.item, "Data"), archiveKey(t, p, db.item) db.replace = "refuse" if _, err := d.Supersede(ctx, next, "first"); err == nil || db.replaced != 0 || value(db.item, "PlanID") != p.ID || value(db.item, "Owner") != "" || len(db.archived) != 1 { t.Fatal("interrupted handover left an unexpected journal", err) @@ -287,14 +416,17 @@ func TestHandoverSurvivesLostResponsesAndRepeats(t *testing.T) { t.Fatal("retired operation lost after an interrupted handover", err) } db.replace = "" - if _, err := d.Supersede(ctx, next, "second"); err != nil || db.replaced != 1 || len(db.archived) != 1 || value(db.archived[p.ArchiveKey()], "Data") != before || value(db.item, "Owner") != "second" { + if _, err := d.Supersede(ctx, next, "second"); err != nil || db.replaced != 1 || len(db.archived) != 1 || value(db.archived[archive], "Data") != before || value(db.item, "Owner") != "second" { t.Fatal("repeat after an interrupted handover failed", err) } }) - t.Run("archive-holds-something-else", func(t *testing.T) { + t.Run("archive-of-this-revision-holds-something-else", func(t *testing.T) { + // Same plan, same revision, other data: no checkpoint of this tool does + // that. Nothing is overwritten and nothing is handed over. d, db, p, next, _ := retiredFixture(t) - db.archived = map[string]map[string]types.AttributeValue{p.ArchiveKey(): {"Network": str(p.ArchiveKey()), "PlanID": str(p.ID), "Data": str("{}"), "Revision": number(1)}} - if _, err := d.Supersede(context.Background(), next, "runner"); err == nil || db.replaced != 0 || value(db.item, "PlanID") != p.ID || value(db.archived[p.ArchiveKey()], "Data") != "{}" { + archive := archiveKey(t, p, db.item) + db.archived = map[string]map[string]types.AttributeValue{archive: {"Network": str(archive), "PlanID": str(p.ID), "Data": str("{}"), "Revision": db.item["Revision"]}} + if _, err := d.Supersede(context.Background(), next, "runner"); err == nil || db.replaced != 0 || value(db.item, "PlanID") != p.ID || value(db.archived[archive], "Data") != "{}" { t.Fatal("handover overwrote or ignored different archived evidence", err) } }) @@ -338,6 +470,7 @@ func TestSuccessorIsItselfRetiredAndSucceeded(t *testing.T) { if err != nil { t.Fatal(err) } + first := r.Predecessor.Archive // The second fleet is provisioned, destroyed and retired in its turn. for j, target := range next.Targets { at := r.CreatedAt @@ -364,7 +497,7 @@ func TestSuccessorIsItselfRetiredAndSucceeded(t *testing.T) { t.Fatal("second handover failed", err) } // Both earlier operations are kept, each under its own archive key. - if len(db.archived) != 2 || value(db.archived[p.ArchiveKey()], "PlanID") != p.ID || value(db.archived[next.ArchiveKey()], "PlanID") != next.ID { + if len(db.archived) != 2 || fresh.Predecessor.Archive == first || value(db.archived[first], "PlanID") != p.ID || value(db.archived[fresh.Predecessor.Archive], "PlanID") != next.ID { t.Fatal("an archived operation was lost or replaced") } for _, old := range []provision.Plan{p, next} { diff --git a/internal/provision/plan.go b/internal/provision/plan.go index f39618b..e384fb3 100644 --- a/internal/provision/plan.go +++ b/internal/provision/plan.go @@ -9,6 +9,7 @@ import ( "fmt" "reflect" "sort" + "strconv" "strings" "github.com/dashpay/dash-network-go/internal/spec" @@ -108,9 +109,14 @@ func (p Plan) Key() string { return p.Network.AWS.AccountID + "/" + p.Network.AWS.Region + "/" + p.Network.Metadata.Name } -// ArchiveKey is where the journal keeps this plan's operation once its network -// key has been reused. A network name cannot contain '#'. -func (p Plan) ArchiveKey() string { return p.Key() + "#retired#" + p.ID } +// ArchiveKey is where the journal keeps this plan's operation, as it stood at +// the given revision, once its network key has been reused. The revision is +// part of the key so that an archive is never compared with, or blocked by, a +// copy of some other state of the same operation. A network name cannot +// contain '#'. +func (p Plan) ArchiveKey(revision int64) string { + return p.Key() + "#retired#" + p.ID + "#" + strconv.FormatInt(revision, 10) +} func (p Plan) Token(t Target) string { return digest([]string{p.ID, t.Name}) } func (p Plan) Tags(t Target) map[string]string { return map[string]string{ diff --git a/internal/provision/reuse_test.go b/internal/provision/reuse_test.go index 18f6418..aa49a4d 100644 --- a/internal/provision/reuse_test.go +++ b/internal/provision/reuse_test.go @@ -58,7 +58,7 @@ func TestDestroyedNetworkNameIsReusedBySuccessorPlan(t *testing.T) { ctx := context.Background() identity := testutil.Identity{Account: p.Network.AWS.AccountID} // The retired plan itself, and the same network planned afresh, stay refused. - if _, err := ipamRun(p, c, s); err == nil || !strings.Contains(err.Error(), "allocation retired") { + if _, err := ipamRun(p, c, s); !errors.Is(err, ErrRetired) || !strings.Contains(err.Error(), "allocation retired") { t.Fatal("retired plan provisioned again", err) } again, err := Prepare(ctx, p.Network, identity, c) @@ -81,12 +81,12 @@ func TestDestroyedNetworkNameIsReusedBySuccessorPlan(t *testing.T) { if _, err = Execute(ctx, next, identity, c, plainStore{s}, "owner", "test", time.Millisecond, nil); err == nil || s.record.Plan.ID != p.ID { t.Fatal("a journal without succession handed the name over", err) } - old := len(c.Instances) + old, retiredAt := len(c.Instances), s.record.Revision r, err := Execute(ctx, next, identity, c, s, "owner", "test", time.Millisecond, nil) if err != nil || r.Phase != "compute-ready" || r.Plan.ID != next.ID || len(c.Instances) != old+len(p.Targets) || len(c.addresses) != len(p.Targets) { t.Fatal("successor did not provision past the terminated predecessor", err) } - if r.Predecessor == nil || r.Predecessor.PlanID != p.ID || r.Predecessor.Archive != p.ArchiveKey() || len(r.Predecessor.Instances) != len(p.Targets) || r.Bootstrap != nil || r.Deployment != nil || r.Runtime != nil || r.Upgrade != nil { + if r.Predecessor == nil || r.Predecessor.PlanID != p.ID || r.Predecessor.Revision != retiredAt || r.Predecessor.Archive != p.ArchiveKey(retiredAt) || len(r.Predecessor.Instances) != len(p.Targets) || r.Bootstrap != nil || r.Deployment != nil || r.Runtime != nil || r.Upgrade != nil { t.Fatal("successor operation is not fresh or lost its predecessor") } for _, target := range next.Targets { @@ -246,7 +246,7 @@ func TestOnlyARetiredOperationHasASuccessor(t *testing.T) { if err != nil { t.Fatal(err) } - for _, change := range []string{"lost", "plan", "archive", "instances", "unsorted"} { + for _, change := range []string{"lost", "plan", "archive", "revision", "instances", "unsorted"} { r := copyRecord(fresh) switch change { case "lost": @@ -255,6 +255,9 @@ func TestOnlyARetiredOperationHasASuccessor(t *testing.T) { r.Predecessor.PlanID = strings.Repeat("a", 64) case "archive": r.Predecessor.Archive = p.Key() + case "revision": + // The archive key names the revision that was replaced. + r.Predecessor.Revision++ case "instances": r.Predecessor.Instances = nil case "unsorted": diff --git a/internal/provision/runner.go b/internal/provision/runner.go index 484e5d2..3b5d557 100644 --- a/internal/provision/runner.go +++ b/internal/provision/runner.go @@ -6,6 +6,7 @@ import ( "fmt" "reflect" "sort" + "strconv" "time" "github.com/dashpay/dash-network-go/internal/inventory" @@ -58,7 +59,7 @@ func (d *Predecessor) validate(p Plan) error { } return nil } - if p.Succeeds == "" || d.PlanID != p.Succeeds || d.Revision < 1 || d.Archive != p.Key()+"#retired#"+d.PlanID || len(d.Instances) == 0 || len(d.Instances) > 100 || !sort.StringsAreSorted(d.Instances) { + if p.Succeeds == "" || d.PlanID != p.Succeeds || d.Revision < 1 || d.Archive != p.Key()+"#retired#"+d.PlanID+"#"+strconv.FormatInt(d.Revision, 10) || len(d.Instances) == 0 || len(d.Instances) > 100 || !sort.StringsAreSorted(d.Instances) { return errors.New("invalid predecessor in operation journal") } for i, id := range d.Instances { @@ -103,7 +104,7 @@ func NewSuccessor(p Plan, retired Record) (Record, error) { return Record{}, err } r := NewRecord(p) - d := &Predecessor{PlanID: retired.Plan.ID, Revision: retired.Revision, Archive: retired.Plan.ArchiveKey(), Instances: []string{}} + d := &Predecessor{PlanID: retired.Plan.ID, Revision: retired.Revision, Archive: retired.Plan.ArchiveKey(retired.Revision), Instances: []string{}} for _, n := range retired.Nodes { d.Instances = append(d.Instances, n.InstanceID) } @@ -198,6 +199,11 @@ type Store interface { Release(context.Context, Plan, string) error } +// ErrRetired refuses every further use of an operation whose addresses were +// released. A retired operation is evidence, and the proof that releases its +// network's name: no stage may claim or change it again. +var ErrRetired = errors.New("allocation retired; IPAM addresses have been released (to reuse the name, generate a new plan with provision-plan)") + // Superseder is a Store that can hand a network key to the plan that succeeds // the retired operation held under it. Supersede is Acquire for such a plan: // it archives the retired operation, then replaces it with the successor's @@ -262,7 +268,7 @@ func Execute(ctx context.Context, p Plan, identity inventory.STS, cloud EC2, sto return } if r.Phase == "addresses-released" { - return result, errors.New("allocation retired; IPAM addresses have been released (to reuse the name, generate a new plan with provision-plan)") + return result, ErrRetired } for _, n := range r.Nodes { if n.Address != nil && (n.Address.Phase == "releasing" || n.Address.Phase == "released") {