From d578b2e13bae1bc6bacf3a8b291fe4934a8ff0ea Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Sun, 18 Oct 2026 06:04:35 -0400 Subject: [PATCH] feat(appjail): watch updated jails stay up, as podman's containers are An AppJail update ended when director said the jail was up, and an app that died on its new image still read as a clean update: appjail keeps a jail up after its app exits (persist, which its start needs), with nothing running inside, and appjail status says running. Each updated jail is now watched for 30 seconds, like podman's containers: it fails if it goes down, if it is started again (its jid changes), or if nothing runs inside it any more, and at the end if its app keeps crashing (s6 in its log, nothing on its port). Failures are [error] lines, which the page reads as a failed update -- and Roll back is there for it now. --- pkg/engine/appjail/director.go | 6 +- pkg/engine/appjail/watch.go | 145 +++++++++++++++++++++++++++++++ pkg/engine/appjail/watch_test.go | 69 +++++++++++++++ test/ui/appjail-update.js | 5 +- 4 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 pkg/engine/appjail/watch.go create mode 100644 pkg/engine/appjail/watch_test.go diff --git a/pkg/engine/appjail/director.go b/pkg/engine/appjail/director.go index 3a30ba4..28f2e8b 100644 --- a/pkg/engine/appjail/director.go +++ b/pkg/engine/appjail/director.go @@ -203,7 +203,8 @@ func directorUpdate(ctx context.Context, s *stack.Stack, jails []svcJail) (io.Re return directorOp(ctx, s, func(w io.Writer) error { return pullNamedImages(ctx, w, jails) }, func(w io.Writer) error { return downDestroy(ctx, stackDir(s.Dir), w) }, - directorUpStep(ctx, s)) + directorUpStep(ctx, s), + func(w io.Writer) error { return watchJails(ctx, w, jails, healthWindow, "the update") }) } // directorUpdateStopped is an update of a stack that is not running: the new @@ -258,7 +259,8 @@ func directorUpdateServices(ctx context.Context, s *stack.Stack, jails []svcJail return fmt.Errorf("not running: %s", strings.Join(down, ", ")) } return nil - }) + }, + func(w io.Writer) error { return watchJails(ctx, w, jails, healthWindow, "the update") }) } // pullNamedImages pulls the image of each service that names one (`from:`). diff --git a/pkg/engine/appjail/watch.go b/pkg/engine/appjail/watch.go new file mode 100644 index 0000000..5cd5f35 --- /dev/null +++ b/pkg/engine/appjail/watch.go @@ -0,0 +1,145 @@ +package appjail + +import ( + "context" + "fmt" + "io" + "os/exec" + "strings" + "time" + + "github.com/daemonless/fjord/pkg/engine" +) + +// healthWindow is how long updated jails are watched, as podman watches its +// containers after an update. +var healthWindow = 30 * time.Second + +// What a watch asks of a jail; swapped by tests. +var ( + jailJID = func(ctx context.Context, jail string) string { + out, err := exec.CommandContext(ctx, "jls", "-j", jail, "jid").Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) + } + jailProcs = func(ctx context.Context, jail string) int { + out, err := exec.CommandContext(ctx, "ps", "-o", "pid=", "-J", jail).Output() + if err != nil { + return 0 + } + return len(strings.Fields(string(out))) + } + jailHealth = serviceHealth +) + +// watchJails says whether updated jails stay up. +// +// A jail is not a container: appjail keeps it up after its app dies (persist, +// which its own start needs), with nothing running inside, and `appjail +// status` still says running (netlab, 2026-10-17). So up alone proves +// nothing. Each jail fails if it goes down or is started again (its jid +// changes), or if nothing runs inside it any more; and at the end, if its app +// keeps crashing (s6 in its log, nothing on its port). A failure is an +// [error] line, which the page reads as a failed update. +func watchJails(ctx context.Context, w io.Writer, jails []svcJail, window time.Duration, after string) error { + type watched struct { + sj svcJail + jid string + failed bool + } + var ws []*watched + for _, sj := range jails { + if jid := jailJID(ctx, sj.jail); jid != "" { + ws = append(ws, &watched{sj: sj, jid: jid}) + } + } + if len(ws) == 0 { + return nil + } + names := make([]string, len(ws)) + for i, x := range ws { + names[i] = x.sj.svc.Name + } + fmt.Fprintf(w, "\n%s%s stay%s up for %s\n", engine.WatchStartPrefix, strings.Join(names, ", "), + map[bool]string{true: "s", false: ""}[len(ws) == 1], window) + // Its own context, so ending the watch early cancels nothing else. + wctx, stop := context.WithCancel(ctx) + defer stop() + engine.WatchStarted(ctx, stop) + ended := func() bool { + if wctx.Err() == nil || ctx.Err() != nil { + return false + } + fmt.Fprintln(w, engine.WatchEndedLine) + return true + } + + deadline := time.Now().Add(window) + for { + live := 0 + for _, x := range ws { + if x.failed { + continue + } + switch jid := jailJID(wctx, x.sj.jail); { + case wctx.Err() != nil: + case jid == "": + fmt.Fprintf(w, "[error] %s's jail went down after %s -- see its logs\n", x.sj.svc.Name, after) + x.failed = true + case jid != x.jid: + fmt.Fprintf(w, "[error] %s's jail started again since %s -- it is crash-looping; see its logs\n", x.sj.svc.Name, after) + x.failed = true + case jailProcs(wctx, x.sj.jail) == 0 && wctx.Err() == nil: + fmt.Fprintf(w, "[error] %s stopped after %s: its jail is up with nothing running inside -- see its logs\n", x.sj.svc.Name, after) + x.failed = true + default: + live++ + } + } + if ended() { + return nil + } + if live == 0 || !time.Now().Before(deadline) { + break + } + select { + case <-wctx.Done(): + ended() + return nil + case <-time.After(2 * time.Second): + } + } + // Running is not serving: s6 restarts a faulting service inside a jail + // that stays up. Ask the app at the end. + for _, x := range ws { + if x.failed { + continue + } + state, detail := jailHealth(wctx, x.sj.jail, x.sj.svc) + if ended() { + return nil + } + if state == "crashed" { + if why, ok := strings.CutPrefix(detail, "the app keeps crashing: "); ok { + fmt.Fprintf(w, "[error] %s keeps crashing since %s: %s\n", x.sj.svc.Name, after, why) + } else { + fmt.Fprintf(w, "[error] %s keeps crashing inside its jail since %s -- see its logs\n", x.sj.svc.Name, after) + } + x.failed = true + } + } + var failed []string + for _, x := range ws { + if x.failed { + failed = append(failed, x.sj.svc.Name) + } else { + fmt.Fprintf(w, "[fjord] %s has stayed up for %s\n", x.sj.svc.Name, window) + } + } + if len(failed) > 0 { + return fmt.Errorf("did not stay up: %s", strings.Join(failed, ", ")) + } + return nil +} diff --git a/pkg/engine/appjail/watch_test.go b/pkg/engine/appjail/watch_test.go new file mode 100644 index 0000000..2d8ebc3 --- /dev/null +++ b/pkg/engine/appjail/watch_test.go @@ -0,0 +1,69 @@ +package appjail + +import ( + "context" + "strings" + "testing" + "time" + + composepkg "github.com/daemonless/fjord/pkg/compose" +) + +// fakeJail answers the watch's three questions from a script. +type fakeJail struct { + jids []string // jid per ask; the last repeats + procs int + health string + asks int +} + +func (f *fakeJail) install(t *testing.T) { + oj, op, oh := jailJID, jailProcs, jailHealth + jailJID = func(context.Context, string) string { + i := min(f.asks, len(f.jids)-1) + f.asks++ + return f.jids[i] + } + jailProcs = func(context.Context, string) int { return f.procs } + jailHealth = func(context.Context, string, composepkg.Service) (string, string) { return f.health, "" } + t.Cleanup(func() { jailJID, jailProcs, jailHealth = oj, op, oh }) +} + +func watch(t *testing.T, f *fakeJail) (string, error) { + f.install(t) + var out strings.Builder + err := watchJails(context.Background(), &out, []svcJail{{svc: composepkg.Service{Name: "web"}, jail: "app_web"}}, 2*time.Second, "the update") + return out.String(), err +} + +func TestWatchJails(t *testing.T) { + for _, c := range []struct { + name string + f fakeJail + says string + fail bool + }{ + {"stays up", fakeJail{jids: []string{"7"}, procs: 9, health: "running"}, "web has stayed up", false}, + {"goes down", fakeJail{jids: []string{"7", ""}, procs: 9}, "jail went down", true}, + {"started again", fakeJail{jids: []string{"7", "8"}, procs: 9}, "started again", true}, + // What appjail does when the app dies: the jail stays, empty. + {"up with nothing inside", fakeJail{jids: []string{"7"}, procs: 0}, "nothing running inside", true}, + {"crash-looping inside", fakeJail{jids: []string{"7"}, procs: 9, health: "crashed"}, "keeps crashing", true}, + } { + out, err := watch(t, &c.f) + if (err != nil) != c.fail || !strings.Contains(out, c.says) { + t.Errorf("%s: err %v\n%s", c.name, err, out) + } + if !strings.Contains(out, "[fjord] checking web stays up for 2s") { + t.Errorf("%s: no start line\n%s", c.name, out) + } + } +} + +// A jail that never came up is the update's to report, not the watch's. +func TestWatchJailsNothingToWatch(t *testing.T) { + out, err := watch(t, &fakeJail{jids: []string{""}}) + if err != nil || out != "" { + t.Fatalf("err %v, out %q", err, out) + } +} diff --git a/test/ui/appjail-update.js b/test/ui/appjail-update.js index 7913778..2c54772 100644 --- a/test/ui/appjail-update.js +++ b/test/ui/appjail-update.js @@ -30,8 +30,11 @@ const detail = async (page) => (await page.request.get(BASE + '/api/stacks/' + N for (let i = 0; i < 100 && !seen; i++) { d = await detail(page); seen = !!d.busy; if (!seen) await sleep(300); } note('the update is seen running', seen, seen ? `busy=${d.busy}` : 'never seen busy'); for (let i = 0; i < 120; i++) { d = await detail(page); if (!d.busy) break; await sleep(3000); } - for (let i = 0; i < 40 && !out; i++) await sleep(500); + // Not busy comes when the stay-up check starts; the stream ends with it, + // 30 seconds on. + for (let i = 0; i < 240 && !out; i++) await sleep(500); note('only sync was rebuilt', /Creating sync/.test(out) && !/(Creating|Stopping|Destroying) web/.test(out), out ? out.split('\n').filter((l) => /^\$|Creating|Destroying|Stopping|Nothing to do/.test(l)).join(' | ').slice(0, 300) : 'no stream captured'); + note('the stay-up check ran on sync', /checking sync stays up for 30s/.test(out) && /sync has stayed up for 30s/.test(out), out ? (out.split('\n').filter((l) => /stays up|stayed up/.test(l)).join(' | ') || 'no stay-up lines') : 'no stream captured'); note('the stream ended without an [error]', !!out && !/\[error\]/.test(out), out ? (out.split('\n').filter((l) => /\[error\]/.test(l)).join(' | ') || 'clean') : 'no stream captured'); d = await detail(page); note('the stack is running, no failure recorded', d.status?.state === 'running' && !d.state?.last_failure, `${d.status?.state}, ${JSON.stringify(d.state?.last_failure || null)}`);