feat: suspend and idempotent resume for sandboxes

.env.example

@@ -39,6 +39,11 @@ SANDBOX_REDIS_URL=redis://127.0.0.1:6379/0
 SANDBOX_LOCAL_ROOT=./data/sandboxes
 SANDBOX_SHARED_ROOT=
 SANDBOX_IDLE_TTL_SECONDS=1800
+SANDBOX_SUSPENDED_RETENTION_SECONDS=2592000
+SANDBOX_SUSPEND_SNAPSHOT=auto
+# A workspace directory this worker no longer owns (resumed on another worker,
+# or released while this worker was away) is deleted after this long; 0 keeps it.
+SANDBOX_ORPHAN_DORMANT_DIR_TTL_SECONDS=86400
 # Concurrent commands per sandbox; only executions with a distinct exec_scope
 # run in parallel, and an unscoped command stays exclusive.
 SANDBOX_MAX_PARALLEL_EXECS_PER_SANDBOX=16

.github/ci/compose.apparmor.yaml

@@ -0,0 +1,13 @@
+# CI only. Layered over compose.yaml on the GitHub-hosted runners, whose Docker
+# enforces its `docker-default` AppArmor profile. That profile denies the
+# `mount` Bubblewrap performs to make `/` a slave mount inside its new mount
+# namespace, so every isolation level fails the startup probe with
+# `bwrap: Failed to make / slave: Permission denied` and the service exits.
+# The Docker Desktop and OrbStack hosts the quick start is written against have
+# no AppArmor, which is why compose.yaml itself does not carry this: the
+# Compose file stays the narrow default, and a host that enforces AppArmor opts
+# in deliberately (docs/ISOLATION.md, "Hosts that enforce AppArmor").
+services:
+  agent-sandbox:
+    security_opt:
+      - apparmor=unconfined

.github/ci/compose.fleet.apparmor.yaml

@@ -0,0 +1,12 @@
+# CI only. See compose.apparmor.yaml: the same Docker AppArmor profile, applied
+# to each replica of the fleet file.
+services:
+  worker-a:
+    security_opt:
+      - apparmor=unconfined
+  worker-b:
+    security_opt:
+      - apparmor=unconfined
+  worker-c:
+    security_opt:
+      - apparmor=unconfined

.github/workflows/ci.yml

@@ -70,6 +70,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Allow unprivileged user namespaces
+        # ubuntu-latest is Ubuntu 24.04, which ships
+        # kernel.apparmor_restrict_unprivileged_userns=1: a process that creates
+        # a user namespace loses its capabilities inside it, so Bubblewrap cannot
+        # write its uid map ("setting up uid map: Permission denied") and no
+        # isolation level can be negotiated. That is the host this job runs on,
+        # not something the code under test can work around; the service reports
+        # it as a preflight warning. Switching the restriction off for the
+        # duration of the job is what the sandbox's own documentation tells an
+        # operator to do on such a host.
+        run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
       - name: Build the base image
         run: docker build -f Dockerfile.base -t agent-sandbox-base:latest .
       - name: Build the test image
@@ -114,13 +125,24 @@ jobs:
         run: python -m pip install uv
       - name: Install project
         run: uv sync --frozen
+      - name: Allow unprivileged user namespaces
+        # ubuntu-latest is Ubuntu 24.04, which ships
+        # kernel.apparmor_restrict_unprivileged_userns=1: a process that creates
+        # a user namespace loses its capabilities inside it, so Bubblewrap cannot
+        # write its uid map ("setting up uid map: Permission denied") and no
+        # isolation level can be negotiated. That is the host this job runs on,
+        # not something the code under test can work around; the service reports
+        # it as a preflight warning. Switching the restriction off for the
+        # duration of the job is what the sandbox's own documentation tells an
+        # operator to do on such a host.
+        run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
       - name: Build the base image
         run: docker build -f Dockerfile.base -t agent-sandbox-base:latest .
       - name: Start replicas over a shared database and registry
         # The third replica runs with a different SANDBOX_PROFILE_HASH, behind the
         # fleet file's `other-profile` profile, so the checks that depend on a
         # fleet that is not uniform have one to look at.
-        run: docker compose -f compose.fleet.yaml --profile other-profile up -d --wait --build
+        run: docker compose -f compose.fleet.yaml -f .github/ci/compose.fleet.apparmor.yaml --profile other-profile up -d --wait --build
       - name: Verify what a fleet promises
         # The single-replica deployment job cannot see any of this: a request
         # forwarded to the replica that owns the sandbox, a template built on one
@@ -132,10 +154,10 @@ jobs:
         run: uv run python scripts/verify-fleet.py --strict --short-graces
       - name: Fleet logs on failure
         if: failure()
-        run: docker compose -f compose.fleet.yaml --profile other-profile logs --tail 200
+        run: docker compose -f compose.fleet.yaml -f .github/ci/compose.fleet.apparmor.yaml --profile other-profile logs --tail 200
       - name: Stop
         if: always()
-        run: docker compose -f compose.fleet.yaml --profile other-profile down -v
+        run: docker compose -f compose.fleet.yaml -f .github/ci/compose.fleet.apparmor.yaml --profile other-profile down -v
 
   package:
     name: Release artifacts
@@ -192,6 +214,14 @@ jobs:
   deployment:
     name: Deployment verification
     runs-on: ubuntu-latest
+    env:
+      # The README's compose.yaml plus the one AppArmor opt-in this runner's
+      # Docker needs; see the file for why it is not in compose.yaml itself.
+      COMPOSE_FILE: compose.yaml:.github/ci/compose.apparmor.yaml
+      # Job level because Compose interpolates the whole file whenever it runs,
+      # including `config --images`, `logs` and `down`, and a missing token is an
+      # error there rather than an empty string.
+      SANDBOX_INTERNAL_TOKEN: ci-verification-token
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -201,23 +231,30 @@ jobs:
         run: python -m pip install uv
       - name: Install project
         run: uv sync --frozen
+      - name: Allow unprivileged user namespaces
+        # ubuntu-latest is Ubuntu 24.04, which ships
+        # kernel.apparmor_restrict_unprivileged_userns=1: a process that creates
+        # a user namespace loses its capabilities inside it, so Bubblewrap cannot
+        # write its uid map ("setting up uid map: Permission denied") and no
+        # isolation level can be negotiated. That is the host this job runs on,
+        # not something the code under test can work around; the service reports
+        # it as a preflight warning. Switching the restriction off for the
+        # duration of the job is what the sandbox's own documentation tells an
+        # operator to do on such a host.
+        run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
       - name: Build the base image
         # The two commands the README gives a first-time reader, in order. The
         # service image is FROM agent-sandbox-base, so `compose up` alone fails
         # on a machine that has never built it.
         run: docker build -f Dockerfile.base -t agent-sandbox-base:latest .
       - name: Start the deployment the README describes
-        env:
-          SANDBOX_INTERNAL_TOKEN: ci-verification-token
         run: docker compose up -d --wait --build
       - name: Scan the image for internal markers
         # What the build context put in the image, rather than the whole
         # filesystem: the base image's keyrings, CA bundle and third-party code
         # produce tens of thousands of findings on their own. The script targets
         # /app and refuses to run when /app holds no application code, so a
         # "clean" answer cannot mean "looked at nothing".
-        env:
-          SANDBOX_INTERNAL_TOKEN: ci-verification-token
         run: ./scripts/scan-image.sh
       - name: Prove the image scan fails on an image that carries one
         # A gate that has only ever been seen pass may be wired to nothing. This
@@ -252,8 +289,6 @@ jobs:
         # --strict so a skipped check fails: a deployment that verified half of
         # what it claims is not verified. This is also the only thing that runs
         # `examples/`, so an example cannot rot into a script that misleads.
-        env:
-          SANDBOX_INTERNAL_TOKEN: ci-verification-token
         run: uv run python scripts/verify-deployment.py --strict
       - name: Deployment logs on failure
         if: failure()

.github/workflows/polyglot.yml

@@ -43,6 +43,17 @@ jobs:
         run: python -m pip install uv
       - name: Install project
         run: uv sync --frozen
+      - name: Allow unprivileged user namespaces
+        # ubuntu-latest is Ubuntu 24.04, which ships
+        # kernel.apparmor_restrict_unprivileged_userns=1: a process that creates
+        # a user namespace loses its capabilities inside it, so Bubblewrap cannot
+        # write its uid map ("setting up uid map: Permission denied") and no
+        # isolation level can be negotiated. That is the host this job runs on,
+        # not something the code under test can work around; the service reports
+        # it as a preflight warning. Switching the restriction off for the
+        # duration of the job is what the sandbox's own documentation tells an
+        # operator to do on such a host.
+        run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
       - name: Free the disk the toolchain images need
         # Three images of roughly two gigabytes each, plus the build cache that
         # produced them, against a runner whose root filesystem also carries an

README.md

@@ -66,6 +66,8 @@ execution-backend SPI for isolate, VM, or remote backends. See
   reusable environments cross workers on, and where a caller keeps checkpoint
   archives. Snapshot and restore are the caller's to orchestrate.
 - Generation fencing so stale workers cannot mutate reassigned workspaces.
+- Suspend and idempotent resume: a waiting agent gives its slot back and keeps
+  its workspace, with a retention period and disk reclamation for dormant ones.
 - REST and stateless Streamable HTTP MCP APIs.
 - Reusable environment templates, built once and mounted read-only everywhere.
 - A dependency-free fleet console at `/admin`: no build step, no CDN.
@@ -481,6 +483,30 @@ File API calls lock only the target path, so reads and writes to different
 files stay concurrent with running commands. Writes go through a temporary
 file and an atomic rename, so a reader never observes a partial file.
 
+## Suspend and resume
+
+An agent waiting on something slow does not need to hold a slot. Suspend
+releases the sandbox's capacity slot and keeps its workspace; resume takes a
+slot again and continues with the same files, idempotently.
+
+```bash
+# Give the slot back while waiting. Refused with 409 while anything runs.
+POST /api/v1/sandboxes/my-agent/suspend   {"generation":3}
+
+# Continue later. Idempotent; use the generation it returns from then on.
+POST /api/v1/sandboxes/my-agent/resume
+```
+
+Suspend and exec admission are decided on the same route row, so a running
+command refuses the suspend (`409 SANDBOX_SUSPEND_BUSY`) and a suspended sandbox
+refuses commands (`409 SANDBOX_SUSPENDED`); other sandboxes and scopes are never
+touched. Resume reuses the directory on the original worker when it can, and
+otherwise moves the sandbox to another worker from a shared workspace or an
+object-store snapshot. Suspended sandboxes are released after
+`SANDBOX_SUSPENDED_RETENTION_SECONDS` (30 days by default). No process is
+frozen and no memory is restored. See [Suspend and resume](docs/SUSPEND_RESUME.md)
+for the state machine, snapshots, and disk reclamation.
+
 ## Environment templates
 
 Every sandbox starts with an empty `/envs`, so an application that needs NumPy,

README.zh-CN.md

@@ -43,6 +43,7 @@ Sandloom 是面向 AI Agent 的高密度执行服务。一个可信的 Worker 
 - 支持内存或 Redis Worker 注册与发现。
 - 支持兼容 S3 的对象存储，使用标准 AWS 凭据链，用于跨 Worker 分发可复用环境以及保存调用方的检查点归档；快照和恢复流程由调用方编排。
 - 通过 generation 隔离令牌，阻止旧 Worker 修改已经重新分配的工作空间。
+- 挂起与幂等恢复：等待中的 Agent 归还名额并保留工作空间，休眠工作空间有保留期限和磁盘回收。
 - 提供 REST 和无状态 Streamable HTTP MCP API。
 - 环境模板构建一次，即可在多个沙箱中以只读方式挂载。
 - `/admin` 提供无依赖的集群管理页面，无需前端构建，也不依赖 CDN。
@@ -308,6 +309,20 @@ POST /api/v1/sandboxes/my-agent/files/move   {"generation":1,"source":"/workspac
 
 文件 API 只锁定目标路径，因此不同文件的读写可以与正在执行的命令并行。写入通过临时文件和原子重命名完成，读取者不会看到写了一半的文件。
 
+## 挂起与恢复
+
+等待耗时任务的 Agent 无需一直占用名额。挂起会释放沙箱的容量名额并保留工作空间；恢复会重新申请名额，并以幂等方式接着使用原来的文件继续工作。
+
+```bash
+# 等待期间归还名额。有命令在运行时返回 409。
+POST /api/v1/sandboxes/my-agent/suspend   {"generation":3}
+
+# 稍后继续。幂等；之后使用返回的 generation。
+POST /api/v1/sandboxes/my-agent/resume
+```
+
+挂起与 exec 准入在同一行路由记录上原子判定：有命令在运行时挂起会被拒绝（`409 SANDBOX_SUSPEND_BUSY`），已挂起的沙箱会拒绝命令（`409 SANDBOX_SUSPENDED`），不会影响其他沙箱或其他 scope。恢复时优先在原 Worker 上复用目录；否则通过共享工作空间或对象存储快照迁移到其他 Worker。挂起的沙箱在 `SANDBOX_SUSPENDED_RETENTION_SECONDS`（默认 30 天）后被释放。不冻结进程，也不恢复内存。状态机、快照和磁盘回收见[挂起与恢复](docs/SUSPEND_RESUME.md)。
+
 ## 环境模板
 
 每个沙箱最初的 `/envs` 都为空，依赖 NumPy、`node_modules` 或额外 Rust 工具链的应用需要重复安装。模板将这些目录构建一次后归档，再以只读方式挂载到需要它的沙箱中。

docs/ADAPTERS.md

@@ -23,8 +23,29 @@ chain. Object storage holds archives rather than filesystem calls: environment
 templates, and checkpoint archives a caller keeps. `BlobStore.upload_checkpoint(sandbox_id, archive)`
 names one at `checkpoints/<sandbox_id>/<archive name>` and returns the URI that
 `get`/`download_to`/`delete` address it by, which is the whole of what this
-project does about checkpoints — it does not snapshot or restore a workspace for
-you. See [Architecture](ARCHITECTURE.md).
+project does about caller checkpoints — it does not snapshot or restore a running
+workspace for you. Suspend is the one exception: it archives a dormant workspace
+under `checkpoints/<sandbox_id>/dormant/` so a resume can land on another worker
+([Suspend and resume](SUSPEND_RESUME.md)). A metadata store opts in to suspend by
+implementing `DormantRouteStore`, and an execution backend by implementing
+`DormantLifecycle`; without them the API answers 501
+`SANDBOX_SUSPEND_UNSUPPORTED`. See [Architecture](ARCHITECTURE.md).
+
+### S3-compatible stores
+
+The adapter speaks plain S3 with path-style addressing and signature v4, so any
+compatible store works. Two have been run against it end to end: LocalStack and
+RustFS (`rustfs/rustfs`). RustFS needs its access-key and secret-key
+variables set, a data directory writable by UID 10001, and its API port
+(9000) as `BLOBSTORE_ENDPOINT`; the bucket does not need to exist, because the
+workers create it at startup. Against RustFS, the following behaved exactly as
+against AWS S3 and needed no change: put/get/head/delete, deleting a missing key
+(204), a multipart upload of a 20 MB object (ETag `<hash>-<parts>`), listing with
+continuation across pages and past 1,000 keys, presigned GET and PUT,
+`If-None-Match: *` (a second write answers 412), a mismatched client region,
+default and `when_required` boto3 checksum modes, and a wrong secret (403).
+Virtual-host addressing was not exercised, because the endpoint was an IP
+address.
 
 ## Plugin discovery
 

docs/ARCHITECTURE.md

@@ -32,8 +32,9 @@ flowchart LR
 - The execution backend creates short-lived isolated processes over a stable
   workspace.
 - The workspace is local or shared POSIX storage. The object store holds
-  environment templates and any checkpoint archives a caller puts there; the
-  service itself neither snapshots nor restores a workspace.
+  environment templates, any checkpoint archives a caller puts there, and the
+  snapshot of a suspended sandbox, which is the only workspace the service
+  itself archives and restores ([Suspend and resume](SUSPEND_RESUME.md)).
 
 ## Sandbox lifecycle
 
@@ -43,7 +44,10 @@ flowchart LR
 3. Every `exec` obtains a workspace lock, creates a fresh Bubblewrap process,
    and persists an idempotency record.
 4. The route moves through `ASSIGNED`, `READY`, `RUNNING`, `RELEASING`, and
-   `RELEASED` states using compare-and-set updates.
+   `RELEASED` states using compare-and-set updates. An idle `READY` route can
+   also be parked as `SUSPENDING`/`SUSPENDED`, which releases its capacity slot
+   and keeps its workspace until a resume; see
+   [Suspend and resume](SUSPEND_RESUME.md).
 5. Reassignment increments `generation`. Stale generations are rejected at
    both the control API and worker API.
 
@@ -67,6 +71,15 @@ sandbox is pinned rather than evicted:
   its execution record is marked `WORKER_LOST` with it. The generation advances,
   so the replacement worker — and any client still holding the old generation —
   cannot mistake the reclaimed sandbox for the one that was running.
+- **Suspended past retention.** A route suspended for longer than
+  `SANDBOX_SUSPENDED_RETENTION_SECONDS` (30 days by default) is released
+  with reason `SUSPEND_EXPIRED`, deleting its directory and snapshot. Under
+  disk pressure the same cycle first drops local copies of suspended sandboxes
+  that have a snapshot. See [Suspend and resume](SUSPEND_RESUME.md).
+- **Directories a worker no longer owns.** A local workspace directory whose
+  route is released or names another worker (a resume landed elsewhere) is
+  deleted by its worker after `SANDBOX_ORPHAN_DORMANT_DIR_TTL_SECONDS`
+  (24 hours by default), timed by a marker on disk so a restart keeps the clock.
 
 A client that was waiting on the execution sees its connection to that worker
 drop. That is not recoverable at the point of failure: the guarantee is that
@@ -181,7 +194,8 @@ a rollback has to stay possible.
 - A shared POSIX workspace has one logical writer. An advisory filesystem lock
   protects the failover overlap window.
 - Local storage failover preserves the logical sandbox ID but not physical
-  files. Callers needing recovery must restore from a checkpoint.
+  files. Callers needing recovery must restore from a checkpoint; a suspended
+  sandbox with a snapshot is the exception, and resumes on another worker.
 
 ## Non-goals