From 344ed8c04f5def181493a2ed21e76a7e08bfc3ac Mon Sep 17 00:00:00 2001 From: Gal Elharar Date: Mon, 19 Oct 2026 15:15:08 +0300 Subject: [PATCH 1/3] fix(server): delegate cgroup controllers to agent scopes Agent systemd scopes only inherited memory/pids, so rootless kind/Podman (and similar nested cgroups) failed inside agent sessions. Enable Delegate on parent/agent slices and scope units so cpu/cpuset/io are available to nested containers. Co-authored-by: Cursor --- apps/server/src/process/agentScope.test.ts | 21 +++++++++ apps/server/src/process/agentScope.ts | 55 ++++++++++++++++++++-- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 30be9346dc1d4..0cfc5d9f55963 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -194,4 +194,25 @@ describe("AgentScope service", () => { }), ), ); + + it.effect("delegates cgroup controllers to agent scopes and slices", () => + withFakeSystemd(({ scope, calls, stateDir }) => + Effect.gen(function* () { + const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + expect(agent.args).toContain("--property=Delegate=yes"); + expect(agent.args).toContain("--property=DelegateControllers=cpu cpuset io memory pids"); + expect(agent.args).toContain("--property=OOMPolicy=stop"); + + const setProperty = calls.filter(([, verb]) => verb === "set-property"); + const delegatedSlices = setProperty + .filter((args) => args.includes("Delegate=yes")) + .map((args) => args[3]); + expect(delegatedSlices).toEqual([ + "app-t3code.slice", + "app-t3code-agents.slice", + ownedAgentSlice(stateDir), + ]); + }), + ), + ); }); diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index 74858468d1b8a..20c8f0ae37d4d 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -85,6 +85,20 @@ function parseCgroupCounter(text: string, key: string): number | undefined { // A stop sends SIGTERM, then SIGKILL after this long. systemd's 90 s default // would outlast the wait for a previous server's agents below. const STOP_TIMEOUT_OPTION = "--property=TimeoutStopSec=10s"; +// Nested agent scopes inherit only the controllers enabled on their parent +// slices. Without Delegate, rootless containers (kind + Podman, Docker rootless) +// fail cgroup setup when tools like make deploy run inside an agent session. +const DELEGATE_OPTIONS = [ + "--property=Delegate=yes", + "--property=DelegateControllers=cpu cpuset io memory pids", +] as const; +const DELEGATE_PROPERTIES = [ + "Delegate=yes", + "DelegateControllers=cpu cpuset io memory pids", +] as const; +// systemd derives parent slice names from the leaf name, so +// app-t3code-agents.slice always lives under app-t3code.slice. +const PARENT_SLICE = "app-t3code.slice"; const STOPPING_POLL_INTERVAL = "100 millis"; const STOPPING_POLL_ATTEMPTS = 20; const CLEAR_FAILED_INTERVAL = "1 second"; @@ -167,9 +181,25 @@ export const make = Effect.gen(function* () { "--", "true", ]).pipe(Effect.map((result) => result._tag === "Some" && result.value.code === 0)); - // Scopes take OOMPolicy from systemd 253; older managers still get scopes. - const supportsOomPolicy = yield* probe([STOP_TIMEOUT_OPTION, "--property=OOMPolicy=stop"]); - if (!supportsOomPolicy && !(yield* probe([STOP_TIMEOUT_OPTION]))) { + // Stop the whole scope when the kernel kills one of its processes, and + // record the result as oom-kill. Scopes take OOMPolicy from systemd 253; + // older user managers still get scopes, without the OOM label. + // Prefer Delegate-enabled options so nested cgroups can enable cpu/cpuset/io. + // Fall back when the local systemd rejects those properties. + const stopOptions = [STOP_TIMEOUT_OPTION]; + const stopWithDelegate = [...stopOptions, ...DELEGATE_OPTIONS]; + const oomOptions = [...stopOptions, "--property=OOMPolicy=stop"]; + const oomWithDelegate = [...oomOptions, ...DELEGATE_OPTIONS]; + const scopeOptions = (yield* probe(oomWithDelegate)) + ? oomWithDelegate + : (yield* probe(oomOptions)) + ? oomOptions + : (yield* probe(stopWithDelegate)) + ? stopWithDelegate + : (yield* probe(stopOptions)) + ? stopOptions + : undefined; + if (scopeOptions === undefined) { yield* Effect.logInfo("Agent scopes are off: no systemd user manager"); return undefined; } @@ -187,10 +217,27 @@ export const make = Effect.gen(function* () { stderr: limited._tag === "Some" ? limited.value.stderr.trim() : undefined, }); } + // Delegate on parent and agent slices so controllers reach nested scopes. + for (const slice of [PARENT_SLICE, AGENT_SLICE, ownedSlice]) { + const delegated = yield* run(systemctl, [ + "--user", + "set-property", + "--runtime", + slice, + ...DELEGATE_PROPERTIES, + ]); + if (delegated._tag === "None" || delegated.value.code !== 0) { + yield* Effect.logWarning("Could not enable cgroup Delegate on agents slice", { + slice, + stderr: delegated._tag === "Some" ? delegated.value.stderr.trim() : undefined, + }); + } + } yield* Effect.logInfo("Agent scopes are on", { slice: AGENT_SLICE, ...limits, - oomPolicy: supportsOomPolicy, + oomPolicy: scopeOptions.includes("--property=OOMPolicy=stop"), + delegate: scopeOptions.includes("--property=Delegate=yes"), }); return { systemdRun, From cd8fee7fad4af81d89f4cc9ea54ddd7829bed107 Mon Sep 17 00:00:00 2001 From: Gal Elharar Date: Mon, 19 Oct 2026 16:35:28 +0300 Subject: [PATCH 2/3] fix(server): use Delegate= controller list on agent scopes systemd-run rejects DelegateControllers= and slices reject Delegate= via set-property. Pass controllers as Delegate= on scopes only so nested cgroups actually receive cpu/cpuset/io. Part of pingdotgg/t3code#21700 Co-authored-by: Cursor --- apps/server/src/process/agentScope.test.ts | 21 ++++-------- apps/server/src/process/agentScope.ts | 40 +++++----------------- 2 files changed, 16 insertions(+), 45 deletions(-) diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 0cfc5d9f55963..c7f89ef424e58 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -195,23 +195,16 @@ describe("AgentScope service", () => { ), ); - it.effect("delegates cgroup controllers to agent scopes and slices", () => - withFakeSystemd(({ scope, calls, stateDir }) => + it.effect("delegates cgroup controllers on agent scopes", () => + withFakeSystemd(({ scope, calls }) => Effect.gen(function* () { const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); - expect(agent.args).toContain("--property=Delegate=yes"); - expect(agent.args).toContain("--property=DelegateControllers=cpu cpuset io memory pids"); + expect(agent.args).toContain("--property=Delegate=cpu cpuset io memory pids"); expect(agent.args).toContain("--property=OOMPolicy=stop"); - - const setProperty = calls.filter(([, verb]) => verb === "set-property"); - const delegatedSlices = setProperty - .filter((args) => args.includes("Delegate=yes")) - .map((args) => args[3]); - expect(delegatedSlices).toEqual([ - "app-t3code.slice", - "app-t3code-agents.slice", - ownedAgentSlice(stateDir), - ]); + expect(calls.filter((args) => args.includes("Delegate=yes"))).toEqual([]); + expect( + calls.filter((args) => args.some((arg) => arg.startsWith("DelegateControllers="))), + ).toEqual([]); }), ), ); diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index 20c8f0ae37d4d..ab791654b8b6c 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -88,17 +88,9 @@ const STOP_TIMEOUT_OPTION = "--property=TimeoutStopSec=10s"; // Nested agent scopes inherit only the controllers enabled on their parent // slices. Without Delegate, rootless containers (kind + Podman, Docker rootless) // fail cgroup setup when tools like make deploy run inside an agent session. -const DELEGATE_OPTIONS = [ - "--property=Delegate=yes", - "--property=DelegateControllers=cpu cpuset io memory pids", -] as const; -const DELEGATE_PROPERTIES = [ - "Delegate=yes", - "DelegateControllers=cpu cpuset io memory pids", -] as const; -// systemd derives parent slice names from the leaf name, so -// app-t3code-agents.slice always lives under app-t3code.slice. -const PARENT_SLICE = "app-t3code.slice"; +// Pass controllers via Delegate= (systemd-run rejects DelegateControllers=). +// Systemd enables those controllers on ancestor slices when the scope starts. +const DELEGATE_OPTION = "--property=Delegate=cpu cpuset io memory pids"; const STOPPING_POLL_INTERVAL = "100 millis"; const STOPPING_POLL_ATTEMPTS = 20; const CLEAR_FAILED_INTERVAL = "1 second"; @@ -187,9 +179,9 @@ export const make = Effect.gen(function* () { // Prefer Delegate-enabled options so nested cgroups can enable cpu/cpuset/io. // Fall back when the local systemd rejects those properties. const stopOptions = [STOP_TIMEOUT_OPTION]; - const stopWithDelegate = [...stopOptions, ...DELEGATE_OPTIONS]; + const stopWithDelegate = [...stopOptions, DELEGATE_OPTION]; const oomOptions = [...stopOptions, "--property=OOMPolicy=stop"]; - const oomWithDelegate = [...oomOptions, ...DELEGATE_OPTIONS]; + const oomWithDelegate = [...oomOptions, DELEGATE_OPTION]; const scopeOptions = (yield* probe(oomWithDelegate)) ? oomWithDelegate : (yield* probe(oomOptions)) @@ -217,33 +209,18 @@ export const make = Effect.gen(function* () { stderr: limited._tag === "Some" ? limited.value.stderr.trim() : undefined, }); } - // Delegate on parent and agent slices so controllers reach nested scopes. - for (const slice of [PARENT_SLICE, AGENT_SLICE, ownedSlice]) { - const delegated = yield* run(systemctl, [ - "--user", - "set-property", - "--runtime", - slice, - ...DELEGATE_PROPERTIES, - ]); - if (delegated._tag === "None" || delegated.value.code !== 0) { - yield* Effect.logWarning("Could not enable cgroup Delegate on agents slice", { - slice, - stderr: delegated._tag === "Some" ? delegated.value.stderr.trim() : undefined, - }); - } - } yield* Effect.logInfo("Agent scopes are on", { slice: AGENT_SLICE, ...limits, oomPolicy: scopeOptions.includes("--property=OOMPolicy=stop"), - delegate: scopeOptions.includes("--property=Delegate=yes"), + delegate: scopeOptions.includes(DELEGATE_OPTION), }); return { systemdRun, systemctl, runtimeDir, - supportsOomPolicy, + supportsOomPolicy: scopeOptions.includes("--property=OOMPolicy=stop"), + supportsDelegate: scopeOptions.includes(DELEGATE_OPTION), }; }), ); @@ -382,6 +359,7 @@ export const make = Effect.gen(function* () { // unload the unit even when it fails. options: [ STOP_TIMEOUT_OPTION, + ...(scope.supportsDelegate ? [DELEGATE_OPTION] : []), ...(scope.supportsOomPolicy ? [`--property=OOMPolicy=${continuing ? "continue" : "stop"}`] : []), From e0ff4143259a1112ccedb8ce9d736193dbef28ab Mon Sep 17 00:00:00 2001 From: Gal Elharar Date: Tue, 20 Oct 2026 12:37:20 +0300 Subject: [PATCH 3/3] test(server): cover agent scope delegation fallback Part of pingdotgg/t3code#21700 --- apps/server/src/process/agentScope.test.ts | 24 +++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index c7f89ef424e58..01f45e1982f7f 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -42,7 +42,13 @@ const withFakeSystemd = ( readonly calls: ReadonlyArray>; readonly stateDir: string; }) => Effect.Effect, - { oomPolicy = true }: { readonly oomPolicy?: boolean } = {}, + { + oomPolicy = true, + delegate = true, + }: { + readonly oomPolicy?: boolean; + readonly delegate?: boolean; + } = {}, ) => Effect.gen(function* () { const bin = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-scope-")); @@ -68,6 +74,9 @@ const withFakeSystemd = ( if (!oomPolicy && args.some((arg) => arg.startsWith("--property=OOMPolicy="))) { return output("", 1); } + if (!delegate && args.some((arg) => arg.startsWith("--property=Delegate="))) { + return output("", 1); + } const [, verb, unit = ""] = args; if (verb === "reset-failed") units.delete(unit); if (verb !== "show") return output(""); @@ -208,4 +217,17 @@ describe("AgentScope service", () => { }), ), ); + + it.effect("falls back when systemd rejects delegation", () => + withFakeSystemd( + ({ scope }) => + Effect.gen(function* () { + const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + expect(agent.args).not.toContain("--property=Delegate=cpu cpuset io memory pids"); + expect(agent.args).toContain("--property=OOMPolicy=stop"); + expect(agent.args).toContain("--property=TimeoutStopSec=10s"); + }), + { delegate: false }, + ), + ); });