feat(appjail): watch updated jails stay up, as podman's containers are

pkg/engine/appjail/director.go

@@ -203,7 +203,8 @@ func directorUpdate(ctx context.Context, s *stack.Stack, jails []svcJail) (io.Re
 	return directorOp(ctx, s,
 		func(w io.Writer) error { return pullNamedImages(ctx, w, jails) },
 		func(w io.Writer) error { return downDestroy(ctx, stackDir(s.Dir), w) },
-		directorUpStep(ctx, s))
+		directorUpStep(ctx, s),
+		func(w io.Writer) error { return watchJails(ctx, w, jails, healthWindow, "the update") })
 }
 
 // directorUpdateStopped is an update of a stack that is not running: the new
@@ -258,7 +259,8 @@ func directorUpdateServices(ctx context.Context, s *stack.Stack, jails []svcJail
 				return fmt.Errorf("not running: %s", strings.Join(down, ", "))
 			}
 			return nil
-		})
+		},
+		func(w io.Writer) error { return watchJails(ctx, w, jails, healthWindow, "the update") })
 }
 
 // pullNamedImages pulls the image of each service that names one (`from:`).

pkg/engine/appjail/watch.go

@@ -0,0 +1,145 @@
+package appjail
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/daemonless/fjord/pkg/engine"
+)
+
+// healthWindow is how long updated jails are watched, as podman watches its
+// containers after an update.
+var healthWindow = 30 * time.Second
+
+// What a watch asks of a jail; swapped by tests.
+var (
+	jailJID = func(ctx context.Context, jail string) string {
+		out, err := exec.CommandContext(ctx, "jls", "-j", jail, "jid").Output()
+		if err != nil {
+			return ""
+		}
+		return strings.TrimSpace(string(out))
+	}
+	jailProcs = func(ctx context.Context, jail string) int {
+		out, err := exec.CommandContext(ctx, "ps", "-o", "pid=", "-J", jail).Output()
+		if err != nil {
+			return 0
+		}
+		return len(strings.Fields(string(out)))
+	}
+	jailHealth = serviceHealth
+)
+
+// watchJails says whether updated jails stay up.
+//
+// A jail is not a container: appjail keeps it up after its app dies (persist,
+// which its own start needs), with nothing running inside, and `appjail
+// status` still says running (netlab, 2026-10-17). So up alone proves
+// nothing. Each jail fails if it goes down or is started again (its jid
+// changes), or if nothing runs inside it any more; and at the end, if its app
+// keeps crashing (s6 in its log, nothing on its port). A failure is an
+// [error] line, which the page reads as a failed update.
+func watchJails(ctx context.Context, w io.Writer, jails []svcJail, window time.Duration, after string) error {
+	type watched struct {
+		sj     svcJail
+		jid    string
+		failed bool
+	}
+	var ws []*watched
+	for _, sj := range jails {
+		if jid := jailJID(ctx, sj.jail); jid != "" {
+			ws = append(ws, &watched{sj: sj, jid: jid})
+		}
+	}
+	if len(ws) == 0 {
+		return nil
+	}
+	names := make([]string, len(ws))
+	for i, x := range ws {
+		names[i] = x.sj.svc.Name
+	}
+	fmt.Fprintf(w, "\n%s%s stay%s up for %s\n", engine.WatchStartPrefix, strings.Join(names, ", "),
+		map[bool]string{true: "s", false: ""}[len(ws) == 1], window)
+	// Its own context, so ending the watch early cancels nothing else.
+	wctx, stop := context.WithCancel(ctx)
+	defer stop()
+	engine.WatchStarted(ctx, stop)
+	ended := func() bool {
+		if wctx.Err() == nil || ctx.Err() != nil {
+			return false
+		}
+		fmt.Fprintln(w, engine.WatchEndedLine)
+		return true
+	}
+
+	deadline := time.Now().Add(window)
+	for {
+		live := 0
+		for _, x := range ws {
+			if x.failed {
+				continue
+			}
+			switch jid := jailJID(wctx, x.sj.jail); {
+			case wctx.Err() != nil:
+			case jid == "":
+				fmt.Fprintf(w, "[error] %s's jail went down after %s -- see its logs\n", x.sj.svc.Name, after)
+				x.failed = true
+			case jid != x.jid:
+				fmt.Fprintf(w, "[error] %s's jail started again since %s -- it is crash-looping; see its logs\n", x.sj.svc.Name, after)
+				x.failed = true
+			case jailProcs(wctx, x.sj.jail) == 0 && wctx.Err() == nil:
+				fmt.Fprintf(w, "[error] %s stopped after %s: its jail is up with nothing running inside -- see its logs\n", x.sj.svc.Name, after)
+				x.failed = true
+			default:
+				live++
+			}
+		}
+		if ended() {
+			return nil
+		}
+		if live == 0 || !time.Now().Before(deadline) {
+			break
+		}
+		select {
+		case <-wctx.Done():
+			ended()
+			return nil
+		case <-time.After(2 * time.Second):
+		}
+	}
+	// Running is not serving: s6 restarts a faulting service inside a jail
+	// that stays up. Ask the app at the end.
+	for _, x := range ws {
+		if x.failed {
+			continue
+		}
+		state, detail := jailHealth(wctx, x.sj.jail, x.sj.svc)
+		if ended() {
+			return nil
+		}
+		if state == "crashed" {
+			if why, ok := strings.CutPrefix(detail, "the app keeps crashing: "); ok {
+				fmt.Fprintf(w, "[error] %s keeps crashing since %s: %s\n", x.sj.svc.Name, after, why)
+			} else {
+				fmt.Fprintf(w, "[error] %s keeps crashing inside its jail since %s -- see its logs\n", x.sj.svc.Name, after)
+			}
+			x.failed = true
+		}
+	}
+	var failed []string
+	for _, x := range ws {
+		if x.failed {
+			failed = append(failed, x.sj.svc.Name)
+		} else {
+			fmt.Fprintf(w, "[fjord] %s has stayed up for %s\n", x.sj.svc.Name, window)
+		}
+	}
+	if len(failed) > 0 {
+		return fmt.Errorf("did not stay up: %s", strings.Join(failed, ", "))
+	}
+	return nil
+}

pkg/engine/appjail/watch_test.go

@@ -0,0 +1,69 @@
+package appjail
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	composepkg "github.com/daemonless/fjord/pkg/compose"
+)
+
+// fakeJail answers the watch's three questions from a script.
+type fakeJail struct {
+	jids   []string // jid per ask; the last repeats
+	procs  int
+	health string
+	asks   int
+}
+
+func (f *fakeJail) install(t *testing.T) {
+	oj, op, oh := jailJID, jailProcs, jailHealth
+	jailJID = func(context.Context, string) string {
+		i := min(f.asks, len(f.jids)-1)
+		f.asks++
+		return f.jids[i]
+	}
+	jailProcs = func(context.Context, string) int { return f.procs }
+	jailHealth = func(context.Context, string, composepkg.Service) (string, string) { return f.health, "" }
+	t.Cleanup(func() { jailJID, jailProcs, jailHealth = oj, op, oh })
+}
+
+func watch(t *testing.T, f *fakeJail) (string, error) {
+	f.install(t)
+	var out strings.Builder
+	err := watchJails(context.Background(), &out, []svcJail{{svc: composepkg.Service{Name: "web"}, jail: "app_web"}}, 2*time.Second, "the update")
+	return out.String(), err
+}
+
+func TestWatchJails(t *testing.T) {
+	for _, c := range []struct {
+		name string
+		f    fakeJail
+		says string
+		fail bool
+	}{
+		{"stays up", fakeJail{jids: []string{"7"}, procs: 9, health: "running"}, "web has stayed up", false},
+		{"goes down", fakeJail{jids: []string{"7", ""}, procs: 9}, "jail went down", true},
+		{"started again", fakeJail{jids: []string{"7", "8"}, procs: 9}, "started again", true},
+		// What appjail does when the app dies: the jail stays, empty.
+		{"up with nothing inside", fakeJail{jids: []string{"7"}, procs: 0}, "nothing running inside", true},
+		{"crash-looping inside", fakeJail{jids: []string{"7"}, procs: 9, health: "crashed"}, "keeps crashing", true},
+	} {
+		out, err := watch(t, &c.f)
+		if (err != nil) != c.fail || !strings.Contains(out, c.says) {
+			t.Errorf("%s: err %v\n%s", c.name, err, out)
+		}
+		if !strings.Contains(out, "[fjord] checking web stays up for 2s") {
+			t.Errorf("%s: no start line\n%s", c.name, out)
+		}
+	}
+}
+
+// A jail that never came up is the update's to report, not the watch's.
+func TestWatchJailsNothingToWatch(t *testing.T) {
+	out, err := watch(t, &fakeJail{jids: []string{""}})
+	if err != nil || out != "" {
+		t.Fatalf("err %v, out %q", err, out)
+	}
+}

test/ui/appjail-update.js

@@ -30,8 +30,11 @@ const detail = async (page) => (await page.request.get(BASE + '/api/stacks/' + N
     for (let i = 0; i < 100 && !seen; i++) { d = await detail(page); seen = !!d.busy; if (!seen) await sleep(300); }
     note('the update is seen running', seen, seen ? `busy=${d.busy}` : 'never seen busy');
     for (let i = 0; i < 120; i++) { d = await detail(page); if (!d.busy) break; await sleep(3000); }
-    for (let i = 0; i < 40 && !out; i++) await sleep(500);
+    // Not busy comes when the stay-up check starts; the stream ends with it,
+    // 30 seconds on.
+    for (let i = 0; i < 240 && !out; i++) await sleep(500);
     note('only sync was rebuilt', /Creating sync/.test(out) && !/(Creating|Stopping|Destroying) web/.test(out), out ? out.split('\n').filter((l) => /^\$|Creating|Destroying|Stopping|Nothing to do/.test(l)).join(' | ').slice(0, 300) : 'no stream captured');
+    note('the stay-up check ran on sync', /checking sync stays up for 30s/.test(out) && /sync has stayed up for 30s/.test(out), out ? (out.split('\n').filter((l) => /stays up|stayed up/.test(l)).join(' | ') || 'no stay-up lines') : 'no stream captured');
     note('the stream ended without an [error]', !!out && !/\[error\]/.test(out), out ? (out.split('\n').filter((l) => /\[error\]/.test(l)).join(' | ') || 'clean') : 'no stream captured');
     d = await detail(page);
     note('the stack is running, no failure recorded', d.status?.state === 'running' && !d.state?.last_failure, `${d.status?.state}, ${JSON.stringify(d.state?.last_failure || null)}`);