diff --git a/pkg/engine/appjail/director.go b/pkg/engine/appjail/director.go index 3a30ba4..28f2e8b 100644 --- a/pkg/engine/appjail/director.go +++ b/pkg/engine/appjail/director.go @@ -203,7 +203,8 @@ func directorUpdate(ctx context.Context, s *stack.Stack, jails []svcJail) (io.Re return directorOp(ctx, s, func(w io.Writer) error { return pullNamedImages(ctx, w, jails) }, func(w io.Writer) error { return downDestroy(ctx, stackDir(s.Dir), w) }, - directorUpStep(ctx, s)) + directorUpStep(ctx, s), + func(w io.Writer) error { return watchJails(ctx, w, jails, healthWindow, "the update") }) } // directorUpdateStopped is an update of a stack that is not running: the new @@ -258,7 +259,8 @@ func directorUpdateServices(ctx context.Context, s *stack.Stack, jails []svcJail return fmt.Errorf("not running: %s", strings.Join(down, ", ")) } return nil - }) + }, + func(w io.Writer) error { return watchJails(ctx, w, jails, healthWindow, "the update") }) } // pullNamedImages pulls the image of each service that names one (`from:`). diff --git a/pkg/engine/appjail/watch.go b/pkg/engine/appjail/watch.go new file mode 100644 index 0000000..5cd5f35 --- /dev/null +++ b/pkg/engine/appjail/watch.go @@ -0,0 +1,145 @@ +package appjail + +import ( + "context" + "fmt" + "io" + "os/exec" + "strings" + "time" + + "github.com/daemonless/fjord/pkg/engine" +) + +// healthWindow is how long updated jails are watched, as podman watches its +// containers after an update. +var healthWindow = 30 * time.Second + +// What a watch asks of a jail; swapped by tests. +var ( + jailJID = func(ctx context.Context, jail string) string { + out, err := exec.CommandContext(ctx, "jls", "-j", jail, "jid").Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) + } + jailProcs = func(ctx context.Context, jail string) int { + out, err := exec.CommandContext(ctx, "ps", "-o", "pid=", "-J", jail).Output() + if err != nil { + return 0 + } + return len(strings.Fields(string(out))) + } + jailHealth = serviceHealth +) + +// watchJails says whether updated jails stay up. +// +// A jail is not a container: appjail keeps it up after its app dies (persist, +// which its own start needs), with nothing running inside, and `appjail +// status` still says running (netlab, 2026-10-17). So up alone proves +// nothing. Each jail fails if it goes down or is started again (its jid +// changes), or if nothing runs inside it any more; and at the end, if its app +// keeps crashing (s6 in its log, nothing on its port). A failure is an +// [error] line, which the page reads as a failed update. +func watchJails(ctx context.Context, w io.Writer, jails []svcJail, window time.Duration, after string) error { + type watched struct { + sj svcJail + jid string + failed bool + } + var ws []*watched + for _, sj := range jails { + if jid := jailJID(ctx, sj.jail); jid != "" { + ws = append(ws, &watched{sj: sj, jid: jid}) + } + } + if len(ws) == 0 { + return nil + } + names := make([]string, len(ws)) + for i, x := range ws { + names[i] = x.sj.svc.Name + } + fmt.Fprintf(w, "\n%s%s stay%s up for %s\n", engine.WatchStartPrefix, strings.Join(names, ", "), + map[bool]string{true: "s", false: ""}[len(ws) == 1], window) + // Its own context, so ending the watch early cancels nothing else. + wctx, stop := context.WithCancel(ctx) + defer stop() + engine.WatchStarted(ctx, stop) + ended := func() bool { + if wctx.Err() == nil || ctx.Err() != nil { + return false + } + fmt.Fprintln(w, engine.WatchEndedLine) + return true + } + + deadline := time.Now().Add(window) + for { + live := 0 + for _, x := range ws { + if x.failed { + continue + } + switch jid := jailJID(wctx, x.sj.jail); { + case wctx.Err() != nil: + case jid == "": + fmt.Fprintf(w, "[error] %s's jail went down after %s -- see its logs\n", x.sj.svc.Name, after) + x.failed = true + case jid != x.jid: + fmt.Fprintf(w, "[error] %s's jail started again since %s -- it is crash-looping; see its logs\n", x.sj.svc.Name, after) + x.failed = true + case jailProcs(wctx, x.sj.jail) == 0 && wctx.Err() == nil: + fmt.Fprintf(w, "[error] %s stopped after %s: its jail is up with nothing running inside -- see its logs\n", x.sj.svc.Name, after) + x.failed = true + default: + live++ + } + } + if ended() { + return nil + } + if live == 0 || !time.Now().Before(deadline) { + break + } + select { + case <-wctx.Done(): + ended() + return nil + case <-time.After(2 * time.Second): + } + } + // Running is not serving: s6 restarts a faulting service inside a jail + // that stays up. Ask the app at the end. + for _, x := range ws { + if x.failed { + continue + } + state, detail := jailHealth(wctx, x.sj.jail, x.sj.svc) + if ended() { + return nil + } + if state == "crashed" { + if why, ok := strings.CutPrefix(detail, "the app keeps crashing: "); ok { + fmt.Fprintf(w, "[error] %s keeps crashing since %s: %s\n", x.sj.svc.Name, after, why) + } else { + fmt.Fprintf(w, "[error] %s keeps crashing inside its jail since %s -- see its logs\n", x.sj.svc.Name, after) + } + x.failed = true + } + } + var failed []string + for _, x := range ws { + if x.failed { + failed = append(failed, x.sj.svc.Name) + } else { + fmt.Fprintf(w, "[fjord] %s has stayed up for %s\n", x.sj.svc.Name, window) + } + } + if len(failed) > 0 { + return fmt.Errorf("did not stay up: %s", strings.Join(failed, ", ")) + } + return nil +} diff --git a/pkg/engine/appjail/watch_test.go b/pkg/engine/appjail/watch_test.go new file mode 100644 index 0000000..2d8ebc3 --- /dev/null +++ b/pkg/engine/appjail/watch_test.go @@ -0,0 +1,69 @@ +package appjail + +import ( + "context" + "strings" + "testing" + "time" + + composepkg "github.com/daemonless/fjord/pkg/compose" +) + +// fakeJail answers the watch's three questions from a script. +type fakeJail struct { + jids []string // jid per ask; the last repeats + procs int + health string + asks int +} + +func (f *fakeJail) install(t *testing.T) { + oj, op, oh := jailJID, jailProcs, jailHealth + jailJID = func(context.Context, string) string { + i := min(f.asks, len(f.jids)-1) + f.asks++ + return f.jids[i] + } + jailProcs = func(context.Context, string) int { return f.procs } + jailHealth = func(context.Context, string, composepkg.Service) (string, string) { return f.health, "" } + t.Cleanup(func() { jailJID, jailProcs, jailHealth = oj, op, oh }) +} + +func watch(t *testing.T, f *fakeJail) (string, error) { + f.install(t) + var out strings.Builder + err := watchJails(context.Background(), &out, []svcJail{{svc: composepkg.Service{Name: "web"}, jail: "app_web"}}, 2*time.Second, "the update") + return out.String(), err +} + +func TestWatchJails(t *testing.T) { + for _, c := range []struct { + name string + f fakeJail + says string + fail bool + }{ + {"stays up", fakeJail{jids: []string{"7"}, procs: 9, health: "running"}, "web has stayed up", false}, + {"goes down", fakeJail{jids: []string{"7", ""}, procs: 9}, "jail went down", true}, + {"started again", fakeJail{jids: []string{"7", "8"}, procs: 9}, "started again", true}, + // What appjail does when the app dies: the jail stays, empty. + {"up with nothing inside", fakeJail{jids: []string{"7"}, procs: 0}, "nothing running inside", true}, + {"crash-looping inside", fakeJail{jids: []string{"7"}, procs: 9, health: "crashed"}, "keeps crashing", true}, + } { + out, err := watch(t, &c.f) + if (err != nil) != c.fail || !strings.Contains(out, c.says) { + t.Errorf("%s: err %v\n%s", c.name, err, out) + } + if !strings.Contains(out, "[fjord] checking web stays up for 2s") { + t.Errorf("%s: no start line\n%s", c.name, out) + } + } +} + +// A jail that never came up is the update's to report, not the watch's. +func TestWatchJailsNothingToWatch(t *testing.T) { + out, err := watch(t, &fakeJail{jids: []string{""}}) + if err != nil || out != "" { + t.Fatalf("err %v, out %q", err, out) + } +} diff --git a/test/ui/appjail-update.js b/test/ui/appjail-update.js index 7913778..2c54772 100644 --- a/test/ui/appjail-update.js +++ b/test/ui/appjail-update.js @@ -30,8 +30,11 @@ const detail = async (page) => (await page.request.get(BASE + '/api/stacks/' + N for (let i = 0; i < 100 && !seen; i++) { d = await detail(page); seen = !!d.busy; if (!seen) await sleep(300); } note('the update is seen running', seen, seen ? `busy=${d.busy}` : 'never seen busy'); for (let i = 0; i < 120; i++) { d = await detail(page); if (!d.busy) break; await sleep(3000); } - for (let i = 0; i < 40 && !out; i++) await sleep(500); + // Not busy comes when the stay-up check starts; the stream ends with it, + // 30 seconds on. + for (let i = 0; i < 240 && !out; i++) await sleep(500); note('only sync was rebuilt', /Creating sync/.test(out) && !/(Creating|Stopping|Destroying) web/.test(out), out ? out.split('\n').filter((l) => /^\$|Creating|Destroying|Stopping|Nothing to do/.test(l)).join(' | ').slice(0, 300) : 'no stream captured'); + note('the stay-up check ran on sync', /checking sync stays up for 30s/.test(out) && /sync has stayed up for 30s/.test(out), out ? (out.split('\n').filter((l) => /stays up|stayed up/.test(l)).join(' | ') || 'no stay-up lines') : 'no stream captured'); note('the stream ended without an [error]', !!out && !/\[error\]/.test(out), out ? (out.split('\n').filter((l) => /\[error\]/.test(l)).join(' | ') || 'clean') : 'no stream captured'); d = await detail(page); note('the stack is running, no failure recorded', d.status?.state === 'running' && !d.state?.last_failure, `${d.status?.state}, ${JSON.stringify(d.state?.last_failure || null)}`);