From d6eeda3333f44d380fbc8d64e5da5c20ba314db4 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 21 Oct 2026 19:48:47 -0500 Subject: [PATCH 1/5] fix(server): retry agent scopes after systemd misses the probe timeout A probe that timed out counted as "no systemd user manager", and the cached detection kept agent scopes off until the server restarted. Probe failures now fail detection, which clears the cache so the next launch probes again. Answers from systemd stay cached as before. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/process/agentScope.test.ts | 74 ++++++++++++++-------- apps/server/src/process/agentScope.ts | 40 ++++++++---- 2 files changed, 78 insertions(+), 36 deletions(-) diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 2a9998f4cfbf6..aafa02824acb6 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -50,6 +50,7 @@ const withFakeSystemd = ( readonly calls: ReadonlyArray>; readonly stateDir: string; readonly cgroupRoot: string; + readonly manager: { stalled: boolean }; }) => Effect.Effect, // A user manager older than systemd 253 rejects OOMPolicy on scopes. { @@ -65,6 +66,7 @@ const withFakeSystemd = ( const cgroupRoot = NodePath.join(bin, "cgroup"); const units = new Map(); const calls: Array> = []; + const manager = { stalled: false }; const output = (stdout: string, code = 0): ProcessRunner.ProcessRunOutput => ({ stdout, stderr: "", @@ -76,30 +78,38 @@ const withFakeSystemd = ( stderrInvalidUtf8: false, }); const runner = ProcessRunner.ProcessRunner.of({ - run: ({ args }) => - Effect.sync(() => { - calls.push(args); - if (!oomPolicy && args.some((arg) => arg.startsWith("--property=OOMPolicy="))) { - return output("", 1); - } - const [, verb, unit = ""] = args; - if (stopFails && verb === "stop") { - return { - ...output(""), - stderr: `Failed to stop ${args.at(-1)}: Unit not loaded.`, - code: ChildProcessSpawner.ExitCode(5), - }; - } - if (verb === "reset-failed") units.delete(unit); - if (verb !== "show") return output(""); - const state = units.get(unit); - return output( - state === undefined - ? "LoadState=not-found\nActiveState=inactive\nResult=success\n" - : `LoadState=loaded\nActiveState=${state.activeState}\nResult=${state.result}\n` + - `ControlGroup=${state.controlGroup ?? ""}\n`, - ); - }), + run: ({ command, args }) => + manager.stalled + ? Effect.fail( + new ProcessRunner.ProcessTimeoutError({ + command, + argumentCount: args.length, + timeoutMs: 5_000, + }), + ) + : Effect.sync(() => { + calls.push(args); + if (!oomPolicy && args.some((arg) => arg.startsWith("--property=OOMPolicy="))) { + return output("", 1); + } + const [, verb, unit = ""] = args; + if (stopFails && verb === "stop") { + return { + ...output(""), + stderr: `Failed to stop ${args.at(-1)}: Unit not loaded.`, + code: ChildProcessSpawner.ExitCode(5), + }; + } + if (verb === "reset-failed") units.delete(unit); + if (verb !== "show") return output(""); + const state = units.get(unit); + return output( + state === undefined + ? "LoadState=not-found\nActiveState=inactive\nResult=success\n" + : `LoadState=loaded\nActiveState=${state.activeState}\nResult=${state.result}\n` + + `ControlGroup=${state.controlGroup ?? ""}\n`, + ); + }), }); const fileSystem = yield* FileSystem.FileSystem; const scope = yield* make.pipe( @@ -115,7 +125,7 @@ const withFakeSystemd = ( }), ); const { stateDir } = yield* ServerConfig.ServerConfig; - return yield* body({ scope, units, calls, stateDir, cgroupRoot }).pipe( + return yield* body({ scope, units, calls, stateDir, cgroupRoot, manager }).pipe( Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))), ); }).pipe( @@ -305,6 +315,20 @@ describe("AgentScope service", () => { ), ); + it.effect("probes systemd again after its user manager missed the timeout", () => + withFakeSystemd(({ scope, manager }) => + Effect.gen(function* () { + manager.stalled = true; + const first = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + expect(unitOf(first)).toBe(""); + manager.stalled = false; + const second = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + expect(unitOf(second)).not.toBe(""); + expect(second.args).toContain("--property=OOMPolicy=stop"); + }), + ), + ); + it.effect("stops agents a killed server left before the first launch", () => withFakeSystemd(({ scope, units, calls, stateDir }) => Effect.gen(function* () { diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index b9b6b56acb140..f660a822d0112 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -10,6 +10,7 @@ import { } from "@t3tools/shared/AgentScope"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { resolveCommandPath } from "@t3tools/shared/shell"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -154,7 +155,7 @@ export const make = Effect.gen(function* () { } }); - const systemd = yield* Effect.cached( + const [detectSystemd, redetectSystemd] = yield* Effect.cachedInvalidateWithTTL( Effect.gen(function* () { const runtimeDir = hostEnvironment.XDG_RUNTIME_DIR; if (runtimeDir === undefined || runtimeDir.length === 0) return undefined; @@ -162,17 +163,25 @@ export const make = Effect.gen(function* () { const systemctl = yield* resolve("systemctl", hostEnvironment); if (systemdRun === null || systemctl === null) return undefined; yield* stopLeftoverAgents(systemctl); + // Only a probe systemd answered decides. A busy user manager can miss the + // timeout, and that failure must not turn scopes off for the server's life. const probe = (options: ReadonlyArray) => - run(systemdRun, [ - "--user", - "--scope", - "--quiet", - "--collect", - `--slice=${AGENT_SLICE}`, - ...options, - "--", - "true", - ]).pipe(Effect.map((result) => result._tag === "Some" && result.value.code === 0)); + runner + .run({ + command: systemdRun, + args: [ + "--user", + "--scope", + "--quiet", + "--collect", + `--slice=${AGENT_SLICE}`, + ...options, + "--", + "true", + ], + timeout: "5 seconds", + }) + .pipe(Effect.map((result) => result.code === 0)); // Scopes take OOMPolicy from systemd 253; older managers still get scopes. const supportsOomPolicy = yield* probe([...SCOPE_OPTIONS, "--property=OOMPolicy=stop"]); if (!supportsOomPolicy && !(yield* probe(SCOPE_OPTIONS))) { @@ -213,6 +222,15 @@ export const make = Effect.gen(function* () { supportsOomPolicy, }; }), + Duration.infinity, + ); + const systemd = detectSystemd.pipe( + Effect.catch((cause) => + redetectSystemd.pipe( + Effect.andThen(Effect.logWarning("Agent scopes are off until systemd answers", { cause })), + Effect.as(undefined), + ), + ), ); // `shared` scopes serve other threads too: tracked only for OOM diagnosis on old systemd. From dbf0c2d0a270c5959236656be2993cda903a364e Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 21 Oct 2026 20:57:08 -0500 Subject: [PATCH 2/5] test(server): seed systemd stalls before scope detection Set the fake manager stall through withFakeSystemd options before make runs. The timeout regression then covers eager startup detection as well as the current lazy launch path. Co-Authored-By: GPT-6.1 Sol --- apps/server/src/process/agentScope.test.ts | 35 ++++++++++++++-------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index aafa02824acb6..bd2b3bc5f8272 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -56,7 +56,12 @@ const withFakeSystemd = ( { oomPolicy = true, stopFails = false, - }: { readonly oomPolicy?: boolean; readonly stopFails?: boolean } = {}, + stalled = false, + }: { + readonly oomPolicy?: boolean; + readonly stopFails?: boolean; + readonly stalled?: boolean; + } = {}, ) => Effect.gen(function* () { const bin = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-scope-")); @@ -66,7 +71,7 @@ const withFakeSystemd = ( const cgroupRoot = NodePath.join(bin, "cgroup"); const units = new Map(); const calls: Array> = []; - const manager = { stalled: false }; + const manager = { stalled }; const output = (stdout: string, code = 0): ProcessRunner.ProcessRunOutput => ({ stdout, stderr: "", @@ -316,16 +321,22 @@ describe("AgentScope service", () => { ); it.effect("probes systemd again after its user manager missed the timeout", () => - withFakeSystemd(({ scope, manager }) => - Effect.gen(function* () { - manager.stalled = true; - const first = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); - expect(unitOf(first)).toBe(""); - manager.stalled = false; - const second = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); - expect(unitOf(second)).not.toBe(""); - expect(second.args).toContain("--property=OOMPolicy=stop"); - }), + withFakeSystemd( + ({ scope, manager }) => + Effect.gen(function* () { + const first = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + expect(unitOf(first)).toBe(""); + manager.stalled = false; + const second = yield* scope.wrap({ + command: "agent", + args: [], + name: "t", + threadId: "a", + }); + expect(unitOf(second)).not.toBe(""); + expect(second.args).toContain("--property=OOMPolicy=stop"); + }), + { stalled: true }, ), ); From ced9337b022674f84d29d6c8a02cafd3b54f698c Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Fri, 23 Oct 2026 22:14:04 -0500 Subject: [PATCH 3/5] fix(server): a late agent scope failure no longer clears a recovered detection Launches that shared a stalled systemd probe each invalidated the cache when they heard the failure. One that heard it after another launch had already redetected systemd cleared that success, and the next launch ran leftover-agent cleanup again, stopping the live agents in the owned slice. The cache now expires failures itself and keeps successes, so callers no longer invalidate it. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/process/agentScope.test.ts | 50 ++++++++++++++++++---- apps/server/src/process/agentScope.ts | 10 +++-- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index bd2b3bc5f8272..4c9c08505e3d1 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -7,7 +7,9 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, expect, it } from "@effect/vitest"; import { AgentScope } from "@t3tools/shared/AgentScope"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; @@ -50,7 +52,7 @@ const withFakeSystemd = ( readonly calls: ReadonlyArray>; readonly stateDir: string; readonly cgroupRoot: string; - readonly manager: { stalled: boolean }; + readonly manager: { stalled: boolean; timeout: Effect.Effect }; }) => Effect.Effect, // A user manager older than systemd 253 rejects OOMPolicy on scopes. { @@ -71,7 +73,7 @@ const withFakeSystemd = ( const cgroupRoot = NodePath.join(bin, "cgroup"); const units = new Map(); const calls: Array> = []; - const manager = { stalled }; + const manager = { stalled, timeout: Effect.void }; const output = (stdout: string, code = 0): ProcessRunner.ProcessRunOutput => ({ stdout, stderr: "", @@ -85,12 +87,16 @@ const withFakeSystemd = ( const runner = ProcessRunner.ProcessRunner.of({ run: ({ command, args }) => manager.stalled - ? Effect.fail( - new ProcessRunner.ProcessTimeoutError({ - command, - argumentCount: args.length, - timeoutMs: 5_000, - }), + ? manager.timeout.pipe( + Effect.andThen( + Effect.fail( + new ProcessRunner.ProcessTimeoutError({ + command, + argumentCount: args.length, + timeoutMs: 5_000, + }), + ), + ), ) : Effect.sync(() => { calls.push(args); @@ -340,6 +346,34 @@ describe("AgentScope service", () => { ), ); + it.effect("keeps a recovered detection when another launch hears the old failure late", () => + withFakeSystemd( + ({ scope, units, calls, stateDir, manager }) => + Effect.gen(function* () { + const timedOut = yield* Deferred.make(); + manager.timeout = Deferred.await(timedOut); + const launch = (threadId: string) => + scope.wrap({ command: "agent", args: [], name: "t", threadId }); + // Both launches share the stalled probe. The first one launches again, + // and finds systemd answering, before the second sees the failure. + const first = yield* launch("a").pipe( + Effect.andThen(Effect.sync(() => (manager.stalled = false))), + Effect.andThen(launch("a")), + Effect.forkChild({ startImmediately: true }), + ); + const second = yield* launch("b").pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.succeed(timedOut, undefined); + expect(sliceOf(yield* Fiber.join(first))).toBe(ownedAgentSlice(stateDir)); + expect(unitOf(yield* Fiber.join(second))).toBe(""); + + units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" }); + yield* launch("c"); + expect(stopsOf(calls)).toEqual([]); + }), + { stalled: true }, + ), + ); + it.effect("stops agents a killed server left before the first launch", () => withFakeSystemd(({ scope, units, calls, stateDir }) => Effect.gen(function* () { diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index f660a822d0112..5a1f9be610fc3 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -12,6 +12,7 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import { resolveCommandPath } from "@t3tools/shared/shell"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; @@ -155,7 +156,7 @@ export const make = Effect.gen(function* () { } }); - const [detectSystemd, redetectSystemd] = yield* Effect.cachedInvalidateWithTTL( + const detectSystemd = yield* Effect.cachedWithTTL( Effect.gen(function* () { const runtimeDir = hostEnvironment.XDG_RUNTIME_DIR; if (runtimeDir === undefined || runtimeDir.length === 0) return undefined; @@ -222,12 +223,13 @@ export const make = Effect.gen(function* () { supportsOomPolicy, }; }), - Duration.infinity, + // Callers sharing a failed probe hear it at different times, so the cache + // expires failures itself; a late caller cannot clear a newer success. + (exit) => (Exit.isSuccess(exit) ? Duration.infinity : Duration.zero), ); const systemd = detectSystemd.pipe( Effect.catch((cause) => - redetectSystemd.pipe( - Effect.andThen(Effect.logWarning("Agent scopes are off until systemd answers", { cause })), + Effect.logWarning("Agent scopes are off until systemd answers", { cause }).pipe( Effect.as(undefined), ), ), From 4103a35db841f5187f74a8f302e4792804cbf8d9 Mon Sep 17 00:00:00 2001 From: only21mil Date: Sat, 31 Oct 2026 23:14:16 +0000 Subject: [PATCH 4/5] test(server): drop the agent scope retry test the late-failure test covers The late-failure test already stalls systemd, launches, clears the stall and expects the next launch to get the agents slice, and it fails on main's agentScope.ts for the same reason. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/process/agentScope.test.ts | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 4c9c08505e3d1..b79918a369e9c 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -326,26 +326,6 @@ describe("AgentScope service", () => { ), ); - it.effect("probes systemd again after its user manager missed the timeout", () => - withFakeSystemd( - ({ scope, manager }) => - Effect.gen(function* () { - const first = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); - expect(unitOf(first)).toBe(""); - manager.stalled = false; - const second = yield* scope.wrap({ - command: "agent", - args: [], - name: "t", - threadId: "a", - }); - expect(unitOf(second)).not.toBe(""); - expect(second.args).toContain("--property=OOMPolicy=stop"); - }), - { stalled: true }, - ), - ); - it.effect("keeps a recovered detection when another launch hears the old failure late", () => withFakeSystemd( ({ scope, units, calls, stateDir, manager }) => From 164e0a4591ccf762fb266b028b87fc67f344d4d0 Mon Sep 17 00:00:00 2001 From: only21mil Date: Sat, 31 Oct 2026 23:18:23 +0000 Subject: [PATCH 5/5] refactor(server): the scope probe reuses the systemd run helper The probe needs a timeout to fail rather than read as no answer, so split that half out of run instead of inlining a second copy of the call and its 5 second timeout. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/process/agentScope.ts | 30 ++++++++++++--------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index 5a1f9be610fc3..eeee9e9b24ae4 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -118,8 +118,10 @@ export const make = Effect.gen(function* () { Effect.provideService(HostProcess.Platform, platform), ); + const runOrFail = (command: string, args: ReadonlyArray) => + runner.run({ command, args, timeout: "5 seconds" }); const run = (command: string, args: ReadonlyArray) => - runner.run({ command, args, timeout: "5 seconds" }).pipe(Effect.option); + runOrFail(command, args).pipe(Effect.option); const readCgroupFile = (controlGroup: string, file: string) => fileSystem @@ -167,22 +169,16 @@ export const make = Effect.gen(function* () { // Only a probe systemd answered decides. A busy user manager can miss the // timeout, and that failure must not turn scopes off for the server's life. const probe = (options: ReadonlyArray) => - runner - .run({ - command: systemdRun, - args: [ - "--user", - "--scope", - "--quiet", - "--collect", - `--slice=${AGENT_SLICE}`, - ...options, - "--", - "true", - ], - timeout: "5 seconds", - }) - .pipe(Effect.map((result) => result.code === 0)); + runOrFail(systemdRun, [ + "--user", + "--scope", + "--quiet", + "--collect", + `--slice=${AGENT_SLICE}`, + ...options, + "--", + "true", + ]).pipe(Effect.map((result) => result.code === 0)); // Scopes take OOMPolicy from systemd 253; older managers still get scopes. const supportsOomPolicy = yield* probe([...SCOPE_OPTIONS, "--property=OOMPolicy=stop"]); if (!supportsOomPolicy && !(yield* probe(SCOPE_OPTIONS))) {