feat: add Kubernetes workspace scheduling and ephemeral storage.

.changeset/proud-lies-yawn.md

@@ -0,0 +1,5 @@
+---
+"@pstdio/pocketcoder-cli": minor
+---
+
+Add Kubernetes workspace node scheduling and ephemeral storage limits.

docs/deployment.md

@@ -176,6 +176,17 @@ server replica—the connection hub and scheduler are intentionally
 single-active. Workspace Jobs use the unprivileged `pocketcoder-workspace`
 service account, not the controller account.
 
+Use driver-level scheduling settings to keep all workspace and warm-pool Jobs
+on a labeled, tainted node pool:
+
+```sh
+export POCKETCODER_KUBERNETES_NODE_SELECTOR='{"onefin.com/workload":"agent-workspace"}'
+export POCKETCODER_KUBERNETES_TOLERATIONS='[{"key":"onefin.com/workload","operator":"Equal","value":"agent-workspace","effect":"NoSchedule"}]'
+```
+
+The selector must be a JSON object with string values. Tolerations must be a
+JSON array. This release supports only the `NoSchedule` effect.
+
 Memory-backed writable paths are mounted with the template uid/gid. Kubernetes
 Jobs set pod `fsGroup` to the template gid with
 `fsGroupChangePolicy: OnRootMismatch`; Docker tmpfs mounts set `uid`, `gid`,
@@ -188,7 +199,16 @@ context before rollout:
 ```sh
 POCKETCODER_KUBERNETES_CONFORMANCE=1 \
 POCKETCODER_KUBERNETES_NAMESPACE=pocketcoder \
-bun test packages/drivers/src/kubernetes.test.ts
+bun test packages/drivers/src/kubernetes-conformance.test.ts
+```
+
+To test a labeled, tainted workspace node pool, also set
+`POCKETCODER_KUBERNETES_CONFORMANCE_IMAGE` to a digest-pinned image and set the
+two scheduling variables above. Then run:
+
+```sh
+POCKETCODER_KUBERNETES_CONFORMANCE=1 \
+bun test packages/drivers/src/kubernetes-scheduling-conformance.test.ts
 ```
 
 With `POCKETCODER_SECRET_PROVIDER=kubernetes`, a template value
@@ -239,6 +259,8 @@ an application role with connect/usage/DML only.
 | `POCKETCODER_SECRET_ROOT` | required for file secrets | Deployment-owned local secret root |
 | `POCKETCODER_KUBERNETES_NAMESPACE` | `default` | Namespace for Jobs and input Secrets |
 | `POCKETCODER_KUBERNETES_SERVICE_ACCOUNT` | none | Service account assigned to workspace Jobs |
+| `POCKETCODER_KUBERNETES_NODE_SELECTOR` | none | JSON object that selects nodes for workspace and warm-pool Jobs |
+| `POCKETCODER_KUBERNETES_TOLERATIONS` | `[]` | JSON array of `NoSchedule` tolerations for workspace and warm-pool Jobs |
 | `POCKETCODER_KUBERNETES_WORKSPACE_CLAIM` | required for PVC | Claim mounted by server and workspace Jobs |
 | `POCKETCODER_KUBERNETES_WORKSPACE_SUBPATH` | `workspaces` | Opaque allocation prefix in the claim |
 | `POCKETCODER_MAX_RETAINED_BYTES` | `500Gi` | Global checkpoint quota |

docs/templates.md

@@ -40,7 +40,7 @@ command, mount, network, privilege, or driver.
       "idleTimeout": "10m"
     },
     "env": { "HOME": "/home/agent" },
-    "resources": { "cpu": "2", "memory": "2Gi" },
+    "resources": { "cpu": "2", "memory": "2Gi", "ephemeralStorage": "10Gi" },
     "timeouts": { "start": "2m", "maxAge": "2h", "idle": "20m", "disconnectGrace": "5m", "terminateGrace": "15s" },
     "security": {
       "uid": 10001, "gid": 10001, "readOnlyRoot": true,
@@ -87,6 +87,9 @@ command, mount, network, privilege, or driver.
 - **`timeouts`** — `start` (registration + first health), `maxAge` (hard
   lifetime), `idle` (no relay activity and agent not running), `disconnectGrace`
   (supervisor reconnect window), `terminateGrace` (TERM→KILL).
+- **`resources`** — required CPU and memory sizing, plus optional scratch disk
+  sizing in `ephemeralStorage`. Ephemeral storage is Kubernetes-only in this
+  release. The Docker driver accepts the field but does not enforce it.
 - **`security`** — non-root uid/gid (≥1000), read-only root, memory-backed
   writable paths, dropped capabilities, no privilege escalation. Values can
   only be stricter than the defaults, never weaker.

packages/contracts/src/template-resources.test.ts

@@ -0,0 +1,22 @@
+import { expect, test } from "bun:test";
+import { parseTemplateManifest } from "./index";
+
+test("accepts an optional ephemeral storage resource", () => {
+  const parsed = parseTemplateManifest({
+    apiVersion: "pocketcoder.dev/v1alpha1",
+    kind: "Template",
+    metadata: { name: "resources" },
+    spec: {
+      version: "1.0.0",
+      image: `registry.test/agent@sha256:${"a".repeat(64)}`,
+      harness: { command: ["agent"] },
+      resources: { cpu: "2", memory: "2Gi", ephemeralStorage: "10Gi" },
+    },
+  });
+
+  expect(parsed.manifest.spec.resources).toEqual({
+    cpu: "2",
+    memory: "2Gi",
+    ephemeralStorage: "10Gi",
+  });
+});

packages/contracts/src/template-schema.ts

@@ -123,6 +123,10 @@ export const TimeoutsSchema = z.object({
 export const ResourcesSchema = z.object({
   cpu: z.string().regex(/^\d+(\.\d+)?m?$/),
   memory: z.string().regex(/^\d+(Mi|Gi)$/),
+  ephemeralStorage: z
+    .string()
+    .regex(/^\d+(Mi|Gi)$/)
+    .optional(),
 });
 
 const RepositorySchema = z.object({

packages/drivers/src/index.ts

@@ -25,6 +25,11 @@ export {
   KubernetesDriver,
   type KubernetesDriverOptions,
 } from "./kubernetes";
+export {
+  type KubernetesSchedulingOptions,
+  type KubernetesToleration,
+  validateToleration,
+} from "./kubernetes-scheduling";
 export {
   KubernetesSecretResolver,
   type KubernetesSecretResolverOptions,

packages/drivers/src/kubernetes-manifests.test.ts

@@ -0,0 +1,106 @@
+import { describe, expect, test } from "bun:test";
+import type { WarmRuntimeLaunch, WorkspaceLaunch } from "@pstdio/pocketcoder-runtime-core";
+import { warmJobManifest, workspaceJobManifest } from "./kubernetes-manifests";
+
+function templateSpec(ephemeralStorage?: string) {
+  return {
+    image: "registry.example/workspace@sha256:fixture",
+    command: ["/bin/sleep", "3600"],
+    env: {},
+    resources: {
+      cpu: "1",
+      memory: "512Mi",
+      ...(ephemeralStorage ? { ephemeralStorage } : {}),
+    },
+    security: {
+      uid: 10_001,
+      gid: 10_001,
+      writableMemoryPaths: ["/tmp"],
+      readOnlyRoot: true,
+      allowPrivilegeEscalation: false,
+      dropCapabilities: ["ALL"],
+      seccomp: "RuntimeDefault",
+    },
+    network: { mode: "unrestricted" },
+  };
+}
+
+function workspaceLaunch(ephemeralStorage?: string) {
+  return {
+    workspace: {
+      id: "workspace-id",
+      templateDigest: "sha256:template",
+      templateSnapshot: { spec: templateSpec(ephemeralStorage) },
+    },
+    mounts: [],
+    secrets: [],
+  } as unknown as WorkspaceLaunch;
+}
+
+function warmLaunch(ephemeralStorage?: string) {
+  return {
+    runtimeId: "runtime-id",
+    template: { digest: "sha256:template", spec: templateSpec(ephemeralStorage) },
+  } as unknown as WarmRuntimeLaunch;
+}
+
+const options = {
+  imagePullPolicy: "IfNotPresent" as const,
+  nodeSelector: { "onefin.com/workload": "agent-workspace" },
+  tolerations: [
+    {
+      key: "onefin.com/workload",
+      operator: "Equal" as const,
+      value: "agent-workspace",
+      effect: "NoSchedule" as const,
+    },
+  ],
+};
+
+function podSpec(
+  manifest: ReturnType<typeof workspaceJobManifest> | ReturnType<typeof warmJobManifest>,
+) {
+  return manifest.spec.template.spec;
+}
+
+describe("Kubernetes Job manifests", () => {
+  test("adds scheduling fields to workspace and warm Jobs", () => {
+    const manifests = [
+      workspaceJobManifest(workspaceLaunch(), "workspace", "input", "egress", options),
+      warmJobManifest(warmLaunch(), "warm", "input", "egress", options),
+    ];
+
+    for (const manifest of manifests) {
+      expect(podSpec(manifest).nodeSelector).toEqual(options.nodeSelector);
+      expect(podSpec(manifest).tolerations).toEqual(options.tolerations);
+    }
+  });
+
+  test("omits scheduling fields from workspace and warm Jobs by default", () => {
+    const defaults = { imagePullPolicy: "IfNotPresent" as const };
+    const manifests = [
+      workspaceJobManifest(workspaceLaunch(), "workspace", "input", "egress", defaults),
+      warmJobManifest(warmLaunch(), "warm", "input", "egress", defaults),
+    ];
+
+    for (const manifest of manifests) {
+      expect("nodeSelector" in podSpec(manifest)).toBe(false);
+      expect("tolerations" in podSpec(manifest)).toBe(false);
+    }
+  });
+
+  test("maps ephemeral storage for workspace and warm Jobs", () => {
+    const manifests = [
+      workspaceJobManifest(workspaceLaunch("10Gi"), "workspace", "input", "egress", options),
+      warmJobManifest(warmLaunch("10Gi"), "warm", "input", "egress", options),
+    ];
+    const expected = {
+      requests: { cpu: "1", memory: "512Mi", "ephemeral-storage": "10Gi" },
+      limits: { cpu: "1", memory: "512Mi", "ephemeral-storage": "10Gi" },
+    };
+
+    for (const manifest of manifests) {
+      expect(podSpec(manifest).containers[0]?.resources).toEqual(expected);
+    }
+  });
+});

packages/drivers/src/kubernetes-manifests.ts

@@ -9,11 +9,18 @@ import {
   KUBERNETES_POOL_LABEL,
   KUBERNETES_WORKSPACE_LABEL,
 } from "./kubernetes-labels";
+import {
+  type KubernetesToleration,
+  resourceRequirements,
+  schedulingFields,
+} from "./kubernetes-scheduling";
 
 interface ManifestOptions {
   serviceAccountName?: string;
   imagePullPolicy: "Always" | "IfNotPresent" | "Never";
   egressImage?: string;
+  nodeSelector?: Record<string, string>;
+  tolerations?: KubernetesToleration[];
 }
 
 function volumeForMount(mount: RuntimeMountRef, index: number) {
@@ -132,6 +139,7 @@ export function workspaceJobManifest(
         spec: {
           restartPolicy: "Never",
           automountServiceAccountToken: false,
+          ...schedulingFields(options),
           ...(options.serviceAccountName ? { serviceAccountName: options.serviceAccountName } : {}),
           securityContext: {
             fsGroup: spec.security.gid,
@@ -148,10 +156,7 @@ export function workspaceJobManifest(
               env: Object.entries(spec.env)
                 .filter(([, value]) => !value.startsWith("secretRef:"))
                 .map(([name, value]) => ({ name, value })),
-              resources: {
-                requests: { cpu: spec.resources.cpu, memory: spec.resources.memory },
-                limits: { cpu: spec.resources.cpu, memory: spec.resources.memory },
-              },
+              resources: resourceRequirements(spec.resources),
               securityContext: {
                 runAsUser: spec.security.uid,
                 runAsGroup: spec.security.gid,
@@ -210,6 +215,7 @@ export function warmJobManifest(
         spec: {
           restartPolicy: "Never",
           automountServiceAccountToken: false,
+          ...schedulingFields(options),
           ...(options.serviceAccountName ? { serviceAccountName: options.serviceAccountName } : {}),
           securityContext: {
             fsGroup: spec.security.gid,
@@ -224,7 +230,7 @@ export function warmJobManifest(
               imagePullPolicy: options.imagePullPolicy,
               command: spec.command,
               env: Object.entries(spec.env).map(([name, value]) => ({ name, value })),
-              resources: { requests: spec.resources, limits: spec.resources },
+              resources: resourceRequirements(spec.resources),
               securityContext: {
                 runAsUser: spec.security.uid,
                 runAsGroup: spec.security.gid,