diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 94ca314ac1f33..63808b30673fc 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -5,7 +5,7 @@ import * as NodePath from "node:path"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, expect, it } from "@effect/vitest"; -import { AgentScope } from "@t3tools/shared/AgentScope"; +import { AgentScope, agentScopeCommand } from "@t3tools/shared/AgentScope"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -50,6 +50,9 @@ const withFakeSystemd = ( readonly calls: ReadonlyArray>; readonly stateDir: string; readonly cgroupRoot: string; + readonly agentPath: string; + /** systemd-run fails as it does when XDG_RUNTIME_DIR is full. */ + readonly breakScopes: () => void; }) => Effect.Effect, // A user manager older than systemd 253 rejects OOMPolicy on scopes. { @@ -65,6 +68,7 @@ const withFakeSystemd = ( const cgroupRoot = NodePath.join(bin, "cgroup"); const units = new Map(); const calls: Array> = []; + let scopesBroken = false; const output = (stdout: string, code = 0): ProcessRunner.ProcessRunOutput => ({ stdout, stderr: "", @@ -76,9 +80,17 @@ const withFakeSystemd = ( stderrInvalidUtf8: false, }); const runner = ProcessRunner.ProcessRunner.of({ - run: ({ args }) => + run: ({ command, args }) => Effect.sync(() => { calls.push(args); + if (scopesBroken && NodePath.basename(command) === "systemd-run") { + return { + ...output(""), + stderr: + "Failed to start transient scope unit: Unit run-p402236-i4601228.scope not found.", + code: ChildProcessSpawner.ExitCode(1), + }; + } if (!oomPolicy && args.some((arg) => arg.startsWith("--property=OOMPolicy="))) { return output("", 1); } @@ -113,7 +125,11 @@ const withFakeSystemd = ( }), ); const { stateDir } = yield* ServerConfig.ServerConfig; - return yield* body({ scope, units, calls, stateDir, cgroupRoot }).pipe( + const breakScopes = () => { + scopesBroken = true; + }; + const agentPath = NodePath.join(bin, "agent"); + return yield* body({ scope, units, calls, stateDir, cgroupRoot, agentPath, breakScopes }).pipe( Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))), ); }).pipe( @@ -343,6 +359,19 @@ describe("AgentScope service", () => { ); }); + it.effect("launches agents unscoped once systemd can no longer create a scope", () => + withFakeSystemd(({ scope, agentPath, breakScopes }) => + Effect.gen(function* () { + const scoped = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + expect(scoped.args).toContain("--scope"); + + breakScopes(); + const launch = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "b" }); + expect(launch).toEqual(agentScopeCommand({ command: agentPath, args: [] })); + }), + ), + ); + it.effect("leaves a quiet slice alone", () => withFakeSystemd(({ scope, calls }) => Effect.gen(function* () { diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index cc6d3a04121f6..f0b8a445c0dd5 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -13,6 +13,7 @@ import { resolveCommandPath } from "@t3tools/shared/shell"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import type * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as ServerConfig from "../config.ts"; @@ -148,6 +149,21 @@ export const make = Effect.gen(function* () { } }); + const probe = (systemdRun: string, options: ReadonlyArray) => + run(systemdRun, [ + "--user", + "--scope", + "--quiet", + "--collect", + `--slice=${AGENT_SLICE}`, + ...options, + "--", + "true", + ]); + const probeSucceeded = (result: Option.Option) => + result._tag === "Some" && result.value.code === 0; + const OOM_PROBE_OPTIONS = [...SCOPE_OPTIONS, "--property=OOMPolicy=stop"]; + const systemd = yield* Effect.cached( Effect.gen(function* () { const runtimeDir = hostEnvironment.XDG_RUNTIME_DIR; @@ -156,20 +172,9 @@ export const make = Effect.gen(function* () { const systemctl = yield* resolve("systemctl", hostEnvironment); if (systemdRun === null || systemctl === null) return undefined; yield* stopLeftoverAgents(systemctl); - const probe = (options: ReadonlyArray) => - run(systemdRun, [ - "--user", - "--scope", - "--quiet", - "--collect", - `--slice=${AGENT_SLICE}`, - ...options, - "--", - "true", - ]).pipe(Effect.map((result) => result._tag === "Some" && result.value.code === 0)); // Scopes take OOMPolicy from systemd 253; older managers still get scopes. - const supportsOomPolicy = yield* probe([...SCOPE_OPTIONS, "--property=OOMPolicy=stop"]); - if (!supportsOomPolicy && !(yield* probe(SCOPE_OPTIONS))) { + const supportsOomPolicy = probeSucceeded(yield* probe(systemdRun, OOM_PROBE_OPTIONS)); + if (!supportsOomPolicy && !probeSucceeded(yield* probe(systemdRun, SCOPE_OPTIONS))) { yield* Effect.logInfo("Agent scopes are off: no systemd user manager"); return undefined; } @@ -346,6 +351,19 @@ export const make = Effect.gen(function* () { if (resolved === null) return { command, args }; const scope = yield* systemd; if (scope === undefined) return agentScopeCommand({ command: resolved, args }); + // systemd can stop creating scopes after startup, e.g. once XDG_RUNTIME_DIR fills up. + const checked = yield* probe( + scope.systemdRun, + scope.supportsOomPolicy ? OOM_PROBE_OPTIONS : SCOPE_OPTIONS, + ); + if (!probeSucceeded(checked)) { + yield* Effect.logWarning("Launching an agent without a scope: systemd cannot create one", { + exitCode: checked._tag === "Some" ? checked.value.code : undefined, + stderr: checked._tag === "Some" ? checked.value.stderr.trim() : undefined, + }); + if (threadId !== undefined) yield* track(scope.systemctl, threadId, undefined); + return agentScopeCommand({ command: resolved, args }); + } const unit = `t3code-${name}-${NodeCrypto.randomUUID().slice(0, 8)}.scope`; // A continuing shared scope cannot attribute a tool's OOM to one thread. // Clear the launch thread's previous scope instead.