diff --git a/cmd/fjordd/http_error.go b/cmd/fjordd/http_error.go new file mode 100644 index 0000000..1f09806 --- /dev/null +++ b/cmd/fjordd/http_error.go @@ -0,0 +1,30 @@ +package main + +import ( + "errors" + "io/fs" + "net/http" +) + +// httpError is an error that says which status answers it, so a change made +// inside manager.Update can refuse with a 400 while a failed write stays 500. +type httpError struct { + code int + msg string +} + +func (e httpError) Error() string { return e.msg } + +// replyErr answers err: an httpError with its own status, a stack that is not +// there with 404, anything else with 500. +func replyErr(w http.ResponseWriter, err error) { + var he httpError + switch { + case errors.As(err, &he): + http.Error(w, he.msg, he.code) + case errors.Is(err, fs.ErrNotExist): + http.Error(w, "Stack not found", 404) + default: + http.Error(w, err.Error(), 500) + } +} diff --git a/cmd/fjordd/keep_address.go b/cmd/fjordd/keep_address.go index 125f63b..566a7ab 100644 --- a/cmd/fjordd/keep_address.go +++ b/cmd/fjordd/keep_address.go @@ -124,11 +124,14 @@ func (s *server) keepAddresses(ctx context.Context, st *stack.Stack) (kept, skip } return compose } - out, err := s.manager.UpdateCompose(st.Name, pin) + saved, err := s.manager.Update(st.Name, func(fresh *stack.Stack) error { + fresh.Compose = pin(fresh.Compose) + return nil + }) if err != nil { return []string{"could not keep addresses: " + err.Error()}, skipped } - st.Compose = out + st.Compose = saved.Compose if len(kept) == 0 { return nil, skipped } diff --git a/cmd/fjordd/restart_policy.go b/cmd/fjordd/restart_policy.go index 98bb080..5c0cef6 100644 --- a/cmd/fjordd/restart_policy.go +++ b/cmd/fjordd/restart_policy.go @@ -148,19 +148,21 @@ func (s *server) stackRestart(w http.ResponseWriter, r *http.Request, name strin json.NewEncoder(w).Encode(map[string]any{"status": "saved"}) return } - compose, err := composepkg.SetRestart(st.Compose, req) + var old string + st, err = s.manager.Update(name, func(st *stack.Stack) error { + compose, err := composepkg.SetRestart(st.Compose, req) + if err != nil { + return httpError{400, err.Error()} + } + old, st.Compose = st.Compose, compose + return nil + }) if err != nil { - http.Error(w, err.Error(), 400) - return - } - old := st.Compose - st.Compose = compose - if err := s.manager.Save(st); err != nil { - http.Error(w, err.Error(), 500) + replyErr(w, err) return } env := st.EnvMap() - s.recordPending(name, composepkg.ChangedServices(old, env, compose, env)) + s.recordPending(name, composepkg.ChangedServices(old, env, st.Compose, env)) log.Printf("%s: restart policy %s", name, req) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]any{"status": "saved", "composeHash": composeHash(st)}) diff --git a/cmd/fjordd/rollback.go b/cmd/fjordd/rollback.go index 5212db6..b2df720 100644 --- a/cmd/fjordd/rollback.go +++ b/cmd/fjordd/rollback.go @@ -81,29 +81,23 @@ func (s *server) stackRollback(w http.ResponseWriter, r *http.Request, name stri http.Error(w, `send {"services": [...]}`, 400) return } - st, err := s.manager.Get(name) - if err != nil { - http.Error(w, "Stack not found", 404) - return - } - state, err := s.manager.LoadState(name) - if err != nil || state == nil { - http.Error(w, "no state for "+name, 500) - return - } - for _, svc := range req.Services { - rb, ok := state.Rollback[svc] - if !ok { - http.Error(w, fmt.Sprintf("%s has no earlier image recorded for %s", name, svc), 400) - return + if _, err := s.manager.Update(name, func(st *stack.Stack) error { + state, err := s.manager.LoadState(name) + if err != nil || state == nil { + return httpError{500, "no state for " + name} } - if err := setServiceImage(st, svc, withoutDigest(rb.Ref)+"@"+rb.Digest); err != nil { - http.Error(w, err.Error(), 400) - return + for _, svc := range req.Services { + rb, ok := state.Rollback[svc] + if !ok { + return httpError{400, fmt.Sprintf("%s has no earlier image recorded for %s", name, svc)} + } + if err := setServiceImage(st, svc, withoutDigest(rb.Ref)+"@"+rb.Digest); err != nil { + return httpError{400, err.Error()} + } } - } - if err := s.manager.Save(st); err != nil { - http.Error(w, err.Error(), 500) + return nil + }); err != nil { + replyErr(w, err) return } body, _ := json.Marshal(map[string][]string{"services": req.Services}) @@ -127,29 +121,24 @@ func (s *server) stackUnpin(w http.ResponseWriter, r *http.Request, name string) http.Error(w, `send {"service": "..."}`, 400) return } - st, err := s.manager.Get(name) - if err != nil { - http.Error(w, "Stack not found", 404) - return - } - cur := serviceImage(st, req.Service) - if !strings.Contains(cur, "@") { - http.Error(w, req.Service+" is not pinned", 400) - return - } - restore := withoutDigest(cur) - if state, _ := s.manager.LoadState(name); state != nil { - if rb, ok := state.Rollback[req.Service]; ok && rb.Compose != "" && - composepkg.ExpandEnv(rb.Compose, st.EnvMap()) == restore { - restore = rb.Compose + if _, err := s.manager.Update(name, func(st *stack.Stack) error { + cur := serviceImage(st, req.Service) + if !strings.Contains(cur, "@") { + return httpError{400, req.Service + " is not pinned"} } - } - if err := setServiceImage(st, req.Service, restore); err != nil { - http.Error(w, err.Error(), 400) - return - } - if err := s.manager.Save(st); err != nil { - http.Error(w, err.Error(), 500) + restore := withoutDigest(cur) + if state, _ := s.manager.LoadState(name); state != nil { + if rb, ok := state.Rollback[req.Service]; ok && rb.Compose != "" && + composepkg.ExpandEnv(rb.Compose, st.EnvMap()) == restore { + restore = rb.Compose + } + } + if err := setServiceImage(st, req.Service, restore); err != nil { + return httpError{400, err.Error()} + } + return nil + }); err != nil { + replyErr(w, err) return } s.fleet.forget(name) diff --git a/cmd/fjordd/stack_handlers.go b/cmd/fjordd/stack_handlers.go index e630dda..e4d8047 100644 --- a/cmd/fjordd/stack_handlers.go +++ b/cmd/fjordd/stack_handlers.go @@ -886,15 +886,19 @@ func (s *server) stackSetTag(w http.ResponseWriter, r *http.Request, name string return } } - st.Compose, st.Env = newCompose, newEnv - // And where appjail reads it: the service's from:, or the Makejail's - // tag argument, pin included. - if err := directorFollowsCompose(st, []string{body.Service}); err != nil { - http.Error(w, "director: "+err.Error(), 400) - return - } - if err := s.manager.Save(st); err != nil { - http.Error(w, err.Error(), 500) + if _, err := s.manager.Update(name, func(fresh *stack.Stack) error { + if err := unchangedSince(fresh, st); err != nil { + return err + } + fresh.Compose, fresh.Env = newCompose, newEnv + // And where appjail reads it: the service's from:, or the + // Makejail's tag argument, pin included. + if err := directorFollowsCompose(fresh, []string{body.Service}); err != nil { + return httpError{400, "director: " + err.Error()} + } + return nil + }); err != nil { + replyErr(w, err) return } w.Header().Set("Content-Type", "application/json") @@ -929,17 +933,21 @@ func (s *server) stackSetTag(w http.ResponseWriter, r *http.Request, name string return } } - st.Compose, st.Env = newCompose, newEnv - // And where appjail will read it. Setting only the compose left the jail - // on whatever ${tag} defaults to, so the version on screen and the version - // running were different numbers. A pin goes along: the digest, not just - // the tag. - if err := directorFollowsCompose(st, nil); err != nil { - http.Error(w, "director: "+err.Error(), 400) - return - } - if err := s.manager.Save(st); err != nil { - http.Error(w, err.Error(), 500) + if _, err := s.manager.Update(name, func(fresh *stack.Stack) error { + if err := unchangedSince(fresh, st); err != nil { + return err + } + fresh.Compose, fresh.Env = newCompose, newEnv + // And where appjail will read it. Setting only the compose left the + // jail on whatever ${tag} defaults to, so the version on screen and + // the version running were different numbers. A pin goes along: the + // digest, not just the tag. + if err := directorFollowsCompose(fresh, nil); err != nil { + return httpError{400, "director: " + err.Error()} + } + return nil + }); err != nil { + replyErr(w, err) return } w.Header().Set("Content-Type", "application/json") @@ -1438,18 +1446,32 @@ func (s *server) directorSetTag(ctx context.Context, w http.ResponseWriter, st * } img += "@" + digest } - if err := setServiceImage(st, service, img); err != nil { - http.Error(w, err.Error(), 400) - return - } - if err := s.manager.Save(st); err != nil { - http.Error(w, err.Error(), 500) + if _, err := s.manager.Update(st.Name, func(fresh *stack.Stack) error { + if err := unchangedSince(fresh, st); err != nil { + return err + } + if err := setServiceImage(fresh, service, img); err != nil { + return httpError{400, err.Error()} + } + return nil + }); err != nil { + replyErr(w, err) return } w.Header().Set("Content-Type", "application/json") w.Write([]byte(`{"status":"ok"}`)) } +// unchangedSince refuses a change worked out from snap when the stack has +// been saved since. Set tag asks the registry first (up to 30s, outside the +// lock); writing its answer over a save made meanwhile would undo that save. +func unchangedSince(fresh, snap *stack.Stack) error { + if fresh.Compose != snap.Compose || fresh.Env != snap.Env || fresh.Director != snap.Director { + return httpError{http.StatusConflict, fresh.Name + " was changed while fjord looked up that version; try again"} + } + return nil +} + // stackDismissFailure forgets the stack's last failure. Without it Dismiss // lasted only as long as the page: a failure no later action clears (a slow // inspect read as "gone") came back on every reload. diff --git a/cmd/fjordd/stack_update_test.go b/cmd/fjordd/stack_update_test.go new file mode 100644 index 0000000..9be3d44 --- /dev/null +++ b/cmd/fjordd/stack_update_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/daemonless/fjord/pkg/stack" +) + +func unpinFixture(t *testing.T, compose string) *server { + t.Helper() + s := &server{manager: stack.NewManager(t.TempDir())} + if err := s.manager.Save(&stack.Stack{Name: "app", Compose: compose, Env: "TAG=2.0\n"}); err != nil { + t.Fatal(err) + } + return s +} + +func postUnpin(s *server, name, body string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + s.stackUnpin(rec, httptest.NewRequest(http.MethodPost, "/api/stacks/"+name+"/unpin", strings.NewReader(body)), name) + return rec +} + +// Unpin drops the digest, puts back what the operator wrote when the +// rollback record still matches it, and leaves the .env as it was. +func TestStackUnpin(t *testing.T) { + s := unpinFixture(t, "services:\n web:\n image: repo/web:2.0@sha256:abc\n") + s.manager.UpdateState("app", func(st *stack.State) *stack.State { + return &stack.State{SchemaVersion: 1, Rollback: map[string]stack.RollbackImage{"web": {Compose: "repo/web:${TAG}"}}} + }) + if rec := postUnpin(s, "app", `{"service":"web"}`); rec.Code != http.StatusNoContent { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + st, _ := s.manager.Get("app") + if !strings.Contains(st.Compose, "image: repo/web:${TAG}") || st.Env != "TAG=2.0\n" { + t.Errorf("compose:\n%s\nenv %q", st.Compose, st.Env) + } +} + +// What unpin refuses, with the status each gets. +func TestStackUnpinRefuses(t *testing.T) { + s := unpinFixture(t, "services:\n web:\n image: repo/web:2.0\n") + if rec := postUnpin(s, "app", `{"service":"web"}`); rec.Code != 400 || !strings.Contains(rec.Body.String(), "web is not pinned") { + t.Errorf("not pinned: %d %s", rec.Code, rec.Body) + } + if rec := postUnpin(s, "gone", `{"service":"web"}`); rec.Code != 404 { + t.Errorf("no such stack: %d %s", rec.Code, rec.Body) + } +} + +// Set tag looks the version up before it writes; a save that lands in +// between is not written over. +func TestUnchangedSince(t *testing.T) { + snap := &stack.Stack{Name: "app", Compose: "a", Env: "e"} + if err := unchangedSince(&stack.Stack{Name: "app", Compose: "a", Env: "e"}, snap); err != nil { + t.Errorf("unchanged refused: %v", err) + } + err := unchangedSince(&stack.Stack{Name: "app", Compose: "b", Env: "e"}, snap) + if he, ok := err.(httpError); !ok || he.code != http.StatusConflict { + t.Errorf("changed: %v", err) + } +} diff --git a/pkg/stack/stack.go b/pkg/stack/stack.go index d7f0200..63ef36b 100644 --- a/pkg/stack/stack.go +++ b/pkg/stack/stack.go @@ -182,8 +182,8 @@ type Manager struct { // save drops the first one's field. locks sync.Map // fileLocks holds a *sync.Mutex per stack, taken across each write of - // its files, so UpdateCompose's read-change-write cannot interleave with - // a Save. + // its files, so Update's read-change-write cannot interleave with a Save + // or a Delete. fileLocks sync.Map } @@ -276,6 +276,11 @@ func (m *Manager) Save(stack *Stack) error { return ErrInvalidName } defer lockIn(&m.fileLocks, stack.Name)() + return m.save(stack) +} + +// save is Save without the lock, for callers that hold it. +func (m *Manager) save(stack *Stack) error { dir := filepath.Join(m.StacksDir, stack.Name) if err := os.MkdirAll(dir, 0755); err != nil { return fmt.Errorf("failed to create stack directory: %w", err) @@ -339,6 +344,7 @@ func (m *Manager) Delete(name string) error { if !validName.MatchString(name) { return ErrInvalidName } + defer lockIn(&m.fileLocks, name)() dir := filepath.Join(m.StacksDir, name) if _, err := os.Stat(dir); err != nil { return err // NotExist bubbles up so the handler can 404 @@ -534,26 +540,32 @@ func (m *Manager) UpdateState(name string, change func(st *State) *State) error return m.saveState(name, st) } -// UpdateCompose changes a stack's compose.yaml as it is on disk now, under -// the lock Save takes: change gets the compose and returns the one to write -// (the same text writes nothing). For fjord's own edits at the end of a long -// action -- the copy that action loaded can be minutes old, and saving it -// whole put back a compose the user had saved over meanwhile. -func (m *Manager) UpdateCompose(name string, change func(compose string) string) (string, error) { +// Update changes a stack as it is on disk now, under the lock Save and +// Delete take: change gets the stack read inside the lock and edits it in +// place; an error from change writes nothing and is returned as is, and so +// does a change that leaves the files as they were. Every edit to an +// existing stack goes through here rather than Get then Save: a copy loaded +// before a slow step (an install's stay-up watch, a registry call) and saved +// whole afterwards put back whatever was saved meanwhile. +// +// change must not call Save, Update or Delete: the lock is not reentrant. +func (m *Manager) Update(name string, change func(st *Stack) error) (*Stack, error) { if !ValidName(name) { - return "", ErrInvalidName + return nil, ErrInvalidName } defer lockIn(&m.fileLocks, name)() - path := filepath.Join(m.StacksDir, name, "compose.yaml") - old, err := os.ReadFile(path) + st, err := m.Get(name) if err != nil { - return "", err + return nil, err + } + before := *st + if err := change(st); err != nil { + return nil, err } - out := change(string(old)) - if out == string(old) { - return out, nil + if st.Compose == before.Compose && st.Env == before.Env && st.Director == before.Director && st.Makejail == before.Makejail { + return st, nil } - return out, writeFileAtomic(path, []byte(out), 0644) + return st, m.save(st) } // lock takes the stack's state lock and returns its release. diff --git a/pkg/stack/update_test.go b/pkg/stack/update_test.go new file mode 100644 index 0000000..ab1c9d1 --- /dev/null +++ b/pkg/stack/update_test.go @@ -0,0 +1,101 @@ +package stack + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// Changes to one stack at the same time all land, and a Save in the middle +// is not interleaved with any of them: each Update reads the files after the +// previous write. Get then Save lost all but the last. +func TestUpdateConcurrentChangesAllLand(t *testing.T) { + m := NewManager(t.TempDir()) + if err := m.Save(&Stack{Name: "s", Compose: "services: {}\n", Env: "A=1\n"}); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for i := range 20 { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := m.Update("s", func(st *Stack) error { + st.Compose += fmt.Sprintf("# %d\n", i) + return nil + }); err != nil { + t.Error(err) + } + }() + } + wg.Wait() + st, _ := m.Get("s") + for i := range 20 { + if !strings.Contains(st.Compose, fmt.Sprintf("# %d\n", i)) { + t.Errorf("change %d lost:\n%s", i, st.Compose) + } + } + if st.Env != "A=1\n" { + t.Errorf("env %q", st.Env) + } +} + +// A refused change writes nothing, and its error comes back as is. +func TestUpdateErrorWritesNothing(t *testing.T) { + m := NewManager(t.TempDir()) + m.Save(&Stack{Name: "s", Compose: "services: {}\n"}) + refused := errors.New("no") + _, err := m.Update("s", func(st *Stack) error { + st.Compose = "changed\n" + return refused + }) + if err != refused { + t.Fatalf("err %v", err) + } + if st, _ := m.Get("s"); st.Compose != "services: {}\n" { + t.Errorf("written anyway: %q", st.Compose) + } +} + +// keepAddresses runs on every start: with nothing to pin the files are left +// alone, not rewritten. +func TestUpdateUnchangedDoesNotWrite(t *testing.T) { + dir := t.TempDir() + m := NewManager(dir) + m.Save(&Stack{Name: "s", Compose: "services: {}\n"}) + path := filepath.Join(dir, "s", "compose.yaml") + old := time.Now().Add(-time.Hour) + os.Chtimes(path, old, old) + if _, err := m.Update("s", func(*Stack) error { return nil }); err != nil { + t.Fatal(err) + } + if fi, _ := os.Stat(path); !fi.ModTime().Equal(old) { + t.Errorf("rewritten: mtime %v", fi.ModTime()) + } +} + +// An Update after a Delete finds no stack; it does not bring the directory +// back the way a Save of a copy loaded before the delete did. +func TestUpdateAfterDeleteIsNotFound(t *testing.T) { + dir := t.TempDir() + m := NewManager(dir) + m.Save(&Stack{Name: "s", Compose: "services: {}\n"}) + if err := m.Delete("s"); err != nil { + t.Fatal(err) + } + _, err := m.Update("s", func(st *Stack) error { + st.Compose = "back\n" + return nil + }) + if !errors.Is(err, fs.ErrNotExist) { + t.Errorf("err %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "s")); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("stack dir came back: %v", err) + } +}