From b8adf3270d0a13d808f45ea6b04546df2c173353 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 08:10:14 +0000 Subject: [PATCH 1/3] Enforce workspace storage bounds --- Makefile | 1 + contracts/agents-api/admin-api.md | 4 +- contracts/agents-api/core.openapi.yaml | 62 ++- contracts/agents-api/sandbox-deployment.md | 4 +- contracts/agents-api/zh/admin-api.md | 6 +- contracts/agents-api/zh/sandbox-deployment.md | 6 +- docs/configuration.md | 53 +- docs/getting-started/operations.md | 5 +- docs/workspace-provider.md | 30 +- docs/zh/configuration.md | 55 ++- docs/zh/getting-started/operations.md | 7 +- docs/zh/workspace-provider.md | 32 +- go.mod | 1 + go.sum | 4 + .../agents-client/src/generated/core-api.ts | 22 +- scripts/build-core-distribution.sh | 2 + services/core/cmd/server/main.go | 4 +- .../core/internal/api/workspace_storage.go | 50 +- .../internal/api/workspace_storage_test.go | 52 +- .../workspace_configuration.go | 32 ++ .../workspace_configuration_test.go | 32 ++ .../core/internal/db/queries/workspaces.sql | 32 +- services/core/internal/db/sqlc/models.go | 7 + .../core/internal/db/sqlc/workspaces.sql.go | 139 +++++- services/core/internal/deployment/service.go | 13 +- .../workspace_configuration_test.go | 7 +- .../internal/execution/runtime_compute.go | 2 +- .../internal/execution/runtime_lifecycle.go | 2 +- .../internal/execution/runtime_workspace.go | 14 +- .../runtime_workspace_settlement_test.go | 14 +- .../execution/runtime_workspace_test.go | 29 +- .../postgres/workspacepg/capacity_test.go | 200 ++++++++ .../persistence/postgres/workspacepg/store.go | 183 +++++-- .../postgres/workspacepg/store_test.go | 69 ++- services/core/internal/workspacefs/nfs/nfs.go | 92 +++- .../internal/workspacefs/nfs/nfs_linux.go | 57 ++- .../workspacefs/nfs/nfs_linux_test.go | 64 ++- .../internal/workspacefs/nfs/nfs_other.go | 8 +- .../core/internal/workspacefs/nfs/quota.go | 238 +++++++++ .../workspacefs/nfs/quota/helper_test.py | 278 +++++++++++ .../workspacefs/nfs/quota/oac-workspace-quota | 458 ++++++++++++++++++ .../internal/workspacefs/nfs/quota_test.go | 313 ++++++++++++ .../nfs/testdata/quota/check-request.json | 1 + .../nfs/testdata/quota/check-response.json | 1 + .../nfs/testdata/quota/ensure-request.json | 1 + .../nfs/testdata/quota/ensure-response.json | 1 + .../nfs/testdata/quota/errors.json | 16 + .../workspacefs/providers/registry.go | 55 +-- .../workspacefs/providers/registry_test.go | 34 ++ .../core/internal/workspacefs/workspacefs.go | 37 +- .../core/internal/workspaces/execution.go | 109 +++-- .../internal/workspaces/execution_test.go | 174 ++++++- services/core/internal/workspaces/storage.go | 24 +- services/core/internal/workspaces/types.go | 29 +- .../migrations/000100_workspace_capacity.sql | 68 +++ .../public_handler_fixture_test.go | 12 +- 56 files changed, 2886 insertions(+), 359 deletions(-) create mode 100644 services/core/internal/credentialcrypto/workspace_configuration.go create mode 100644 services/core/internal/credentialcrypto/workspace_configuration_test.go create mode 100644 services/core/internal/persistence/postgres/workspacepg/capacity_test.go create mode 100644 services/core/internal/workspacefs/nfs/quota.go create mode 100644 services/core/internal/workspacefs/nfs/quota/helper_test.py create mode 100755 services/core/internal/workspacefs/nfs/quota/oac-workspace-quota create mode 100644 services/core/internal/workspacefs/nfs/quota_test.go create mode 100644 services/core/internal/workspacefs/nfs/testdata/quota/check-request.json create mode 100644 services/core/internal/workspacefs/nfs/testdata/quota/check-response.json create mode 100644 services/core/internal/workspacefs/nfs/testdata/quota/ensure-request.json create mode 100644 services/core/internal/workspacefs/nfs/testdata/quota/ensure-response.json create mode 100644 services/core/internal/workspacefs/nfs/testdata/quota/errors.json create mode 100644 services/core/internal/workspacefs/providers/registry_test.go create mode 100644 services/core/migrations/000100_workspace_capacity.sql diff --git a/Makefile b/Makefile index 9de40a1a8..61cb1bd82 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,7 @@ check-core-packages: go test $$packages -count=1 -timeout=20m PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s services/core/tests -p 'official_diagnostics_test.py' PYTHONDONTWRITEBYTECODE=1 python3 services/core/deploy/e2b/managed_init_test.py + PYTHONDONTWRITEBYTECODE=1 python3 services/core/internal/workspacefs/nfs/quota/helper_test.py # Integration tests run serially and include bounded lifecycle waits that together exceed Go's 10m default. # OAC_CORE_INTEGRATION_SHARD=INDEX/TOTAL runs one deterministic partition; unset runs them all. diff --git a/contracts/agents-api/admin-api.md b/contracts/agents-api/admin-api.md index 990c436e7..44fce17ea 100644 --- a/contracts/agents-api/admin-api.md +++ b/contracts/agents-api/admin-api.md @@ -42,9 +42,9 @@ Paths are relative to `/core/v1`. ## Workspace storage -Operator scripts use `GET /core/v1/workspace-storage` and `PUT /core/v1/workspace-storage` with a Core key. Web has no workspace storage editor. The request and successful response use the canonical filesystem configuration: `{"id":"","adapter":"","parameters":{...}}`. `parameters` is an adapter-owned JSON object; the selected adapter validates it. Unknown top-level fields are rejected. +Operator scripts use `GET /core/v1/workspace-storage` and `PUT /core/v1/workspace-storage` with a Core key. Web has no workspace storage editor. The successful response is the selection: `{"id":"","adapter":"","parameters":{...},"tenant_capacity_mib":}`. The request has the same required fields and an optional write-only `credential` object. `parameters` and `credential` are adapter-owned JSON objects; the selected adapter validates them. Unknown top-level fields are rejected. -GET returns the selected configuration, or 404 `workspace_storage_not_found` when none is configured. PUT validates and selects an immutable configuration and returns it with 200. Reusing an ID requires identical adapter configuration; changing its meaning or conflicting with retained ownership returns 409 `workspace_storage_conflict`. Configuration changes are serialized with workspace ownership changes, and successful mutations carry the same administrator audit provenance as other deployment settings. The [workspace filesystem protocol](../../docs/workspace-provider.md) owns identity, attachment, admission and retention semantics. +GET returns the selected configuration and tenant capacity policy, or 404 `workspace_storage_not_found` when none is configured. It never returns a credential. PUT validates and selects an immutable configuration together with [`tenant_capacity_mib`](../../docs/configuration.md#tenant-capacity) and returns the selection with 200. A positive `tenant_capacity_mib` requires a configuration that enforces capacity quotas, and such a configuration requires a positive value; otherwise PUT returns 400 `workspace_operation_unsupported`. Core encrypts the credential and never returns, logs or audits it. Reusing an ID requires identical adapter, parameters and credential; omitting `credential` keeps the stored one, and an explicit `null` is invalid. Changing its meaning or conflicting with retained ownership returns 409 `workspace_storage_conflict`. Configuration changes are serialized with workspace ownership changes, and successful mutations carry the same administrator audit provenance as other deployment settings. The [workspace filesystem protocol](../../docs/workspace-provider.md) owns identity, attachment, admission and retention semantics. Invalid configuration returns 400 `invalid_workspace_configuration`; unsupported adapters or combinations return 400 `workspace_operation_unsupported`. Unavailable storage or an unconfirmed operation returns 503 `workspace_storage_unavailable`. Error messages do not expose native paths or adapter error text. The endpoint configures storage independently of Sandbox Provider selection; it does not create or delete a Session workspace. diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index 29303854d..4d30a2ddc 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -830,6 +830,25 @@ definitions: - status - turn_id type: object + api.WorkspaceStorageInput: + properties: + adapter: + type: string + credential: + type: object + id: + format: uuid + type: string + parameters: + type: object + tenant_capacity_mib: + type: integer + required: + - adapter + - id + - parameters + - tenant_capacity_mib + type: object coremetrics.Database: properties: ping_ms: @@ -3836,20 +3855,6 @@ definitions: type: string x-enum-varnames: - AttachmentHostDirectory - workspacefs.Configuration: - properties: - adapter: - type: string - id: - format: uuid - type: string - parameters: - type: object - required: - - adapter - - id - - parameters - type: object workspacefs.Declaration: properties: attachment: @@ -3863,6 +3868,23 @@ definitions: - capacity_quota - user_xattr type: object + workspaces.Selection: + properties: + adapter: + type: string + id: + format: uuid + type: string + parameters: + type: object + tenant_capacity_mib: + type: integer + required: + - adapter + - id + - parameters + - tenant_capacity_mib + type: object writeaudit.APIKey: properties: id: @@ -7432,14 +7454,14 @@ paths: - Core Administration /core/v1/workspace-storage: get: - description: Core key only. Returns the immutable selected filesystem configuration. No configuration returns 404. + description: Core key only. Returns the immutable selected filesystem configuration and the tenant capacity policy, never a credential. No configuration returns 404. produces: - application/json responses: "200": description: OK schema: - $ref: '#/definitions/workspacefs.Configuration' + $ref: '#/definitions/workspaces.Selection' "400": description: Bad Request schema: @@ -7472,21 +7494,21 @@ paths: put: consumes: - application/json - description: Core key only. Selects an immutable filesystem configuration. Reusing an ID requires the same adapter and parameters; conflicting ownership returns 409. + description: Core key only. Selects an immutable filesystem configuration and the tenant capacity policy in one change. Reusing an ID requires the same adapter, parameters and credential; an omitted credential retains the stored one. A positive tenant_capacity_mib requires, and is required by, a configuration that enforces capacity quotas. Conflicting ownership returns 409. parameters: - - description: Immutable workspace storage configuration + - description: Workspace storage selection in: body name: body required: true schema: - $ref: '#/definitions/workspacefs.Configuration' + $ref: '#/definitions/api.WorkspaceStorageInput' produces: - application/json responses: "200": description: OK schema: - $ref: '#/definitions/workspacefs.Configuration' + $ref: '#/definitions/workspaces.Selection' "400": description: Bad Request schema: diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 41668d90e..0149d82b1 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -48,7 +48,7 @@ The request has no Core address. Core derives the deployment's `core_url` from t | `cpus` | Integer, 1 through 255 | | `memory_mib` | Integer, 512 through 1048576 MiB | | `root_disk_mib` | microsandbox: at least 1024 MiB; Docker and E2B: omitted or zero | -| `environment_disk_mib` | microsandbox owned disk: at least 1024 MiB; external filesystem: zero requests no quota; a positive value must be at least 1024 MiB and requires enforced quotas; Docker and E2B: omitted or zero | +| `environment_disk_mib` | microsandbox owned disk: at least 1024 MiB; external filesystem: each new workspace's admitted capacity, where zero requests no quota and a positive value must be at least 1024 MiB and requires enforced quotas; Docker and E2B: omitted or zero | These limits describe each sandbox. A node's `max_active` and `max_retained` are separate reservation limits, and host measurements never permit exceeding either. Native providers may reject values that pass these bounds. @@ -58,7 +58,7 @@ microsandbox configures the CPUs, memory, a managed root disk and a separate own The optional response `specification.workspace` is an immutable capability receipt derived from the selected [workspace filesystem](../../docs/workspace-provider.md), not an independently writable setting. It contains only `attachment`, `user_xattr` and `capacity_quota`; filesystem configuration, paths and credentials stay outside the sandbox specification. Core validates the combined declarations before saving, and retained generation validation and node construction use this receipt. Missing `workspace` preserves owned disk mode, including its existing disk bounds. Create and Resume must provide a binding exactly when the frozen generation selects external storage. -To enable external storage for an existing owned deployment, select the filesystem configuration first, then update the sandbox deployment with compatible resources (`environment_disk_mib: 0` for a filesystem without quotas). Selecting the filesystem does not alter existing generations or allocations. The next sandbox update derives the external receipt; old owned allocations keep their owned disks. A filesystem selection must support the provider's declared requirements, and an already external deployment requires an equivalent declaration with compatible quota semantics. +To enable external storage for an existing owned deployment, select the filesystem configuration first, then update the sandbox deployment with compatible resources (`environment_disk_mib: 0` for a filesystem without quotas). Selecting the filesystem does not alter existing generations or allocations. The next sandbox update derives the external receipt; old owned allocations keep their owned disks. A filesystem selection must support the provider's declared requirements. An already external deployment requires the same `attachment` and `user_xattr`; the selection may change `capacity_quota`, after which new workspaces are refused until a sandbox update sets a matching `environment_disk_mib`. Existing workspaces keep the capacity they were admitted with. ### Runtime release diff --git a/contracts/agents-api/zh/admin-api.md b/contracts/agents-api/zh/admin-api.md index 4650b3d46..8a238fd60 100644 --- a/contracts/agents-api/zh/admin-api.md +++ b/contracts/agents-api/zh/admin-api.md @@ -1,7 +1,7 @@ --- title: "Core 管理 API" source: contracts/agents-api/admin-api.md -source_hash: 9687c874ba9e39dbceeb6f2206aff8e5163309a381262955ca247c67aa10455c +source_hash: 4538ac64f7aee138b27b98f7f33542230d0b21a4fc183dc375fe0803833ac0d6 --- Core 管理 API（`/core/v1`）用于管理安装实例：Project 及其 API 密钥、Project 资源的读取和删除、执行器凭据、部署默认模型、沙箱部署及其节点、监控和审计。Web 的[控制台服务器](../../../docs/zh/web/console-server.md#forwarding-to-core)会为已登录的管理员调用它；运维人员则从 Core 主机上的脚本调用它（[编写 Core API 脚本](../../../docs/zh/getting-started/operations.md#script-the-core-api)）。生成的架构是 [core.openapi.yaml](../core.openapi.yaml)，所有错误都使用 [Core 错误封装](core-errors.md)。 @@ -44,9 +44,9 @@ Core 管理 API（`/core/v1`）用于管理安装实例：Project 及其 API 密 ## 工作区存储 {#workspace-storage} -运维脚本使用 Core key 调用 `GET /core/v1/workspace-storage` 和 `PUT /core/v1/workspace-storage`。Web 没有工作区存储编辑器。请求与成功响应使用规范文件系统配置：`{"id":"","adapter":"","parameters":{...}}`。`parameters` 是适配器定义的 JSON 对象，由所选适配器验证。未知顶层字段会被拒绝。 +运维脚本使用 Core key 调用 `GET /core/v1/workspace-storage` 和 `PUT /core/v1/workspace-storage`。Web 没有工作区存储编辑器。成功响应为选择结果：`{"id":"","adapter":"","parameters":{...},"tenant_capacity_mib":}`。请求包含相同的必需字段，以及可选的只写 `credential` 对象。`parameters` 和 `credential` 是适配器定义的 JSON 对象，由所选适配器验证。未知顶层字段会被拒绝。 -GET 返回所选配置；尚未配置时返回 404 `workspace_storage_not_found`。PUT 验证并选择不可变配置，以 200 返回该配置。复用 ID 要求适配器配置完全一致；更改其含义或与保留归属冲突会返回 409 `workspace_storage_conflict`。配置更改与工作区归属变更串行执行，成功变更携带与其他部署设置相同的管理员审计来源。[工作区文件系统协议](../../../docs/zh/workspace-provider.md) 规定标识、挂载、准入和保留语义。 +GET 返回所选配置和租户容量策略；尚未配置时返回 404 `workspace_storage_not_found`。它绝不返回凭据。PUT 验证并选择不可变配置，同时设置 [`tenant_capacity_mib`](../../../docs/zh/configuration.md#tenant-capacity)，以 200 返回选择结果。正的 `tenant_capacity_mib` 要求实施容量配额的配置，此类配置也要求正值；否则 PUT 返回 400 `workspace_operation_unsupported`。Core 加密凭据，绝不返回、记录日志或审计凭据。复用 ID 要求适配器、参数和凭据完全一致；省略 `credential` 会保留已存储的凭据，显式 `null` 无效。更改其含义或与保留归属冲突会返回 409 `workspace_storage_conflict`。配置更改与工作区归属变更串行执行，成功变更携带与其他部署设置相同的管理员审计来源。[工作区文件系统协议](../../../docs/zh/workspace-provider.md) 规定标识、挂载、准入和保留语义。 无效配置返回 400 `invalid_workspace_configuration`；不支持的适配器或组合返回 400 `workspace_operation_unsupported`。存储不可用或操作尚未确认返回 503 `workspace_storage_unavailable`。错误消息不暴露原生路径或适配器错误文本。此端点独立于 Sandbox Provider 选择配置存储，不创建或删除 Session 工作区。 diff --git a/contracts/agents-api/zh/sandbox-deployment.md b/contracts/agents-api/zh/sandbox-deployment.md index 05fbe1fc8..24b678134 100644 --- a/contracts/agents-api/zh/sandbox-deployment.md +++ b/contracts/agents-api/zh/sandbox-deployment.md @@ -1,7 +1,7 @@ --- title: "沙箱部署" source: contracts/agents-api/sandbox-deployment.md -source_hash: 537ae957a77d625b3adb7828156352a661ae09270b57b174406c6db159a957bd +source_hash: 05d854567f10ddca10d41f14423f0faa68fac4ceebca93dd1bb89129ecd6bf5c --- 沙箱部署为 Core 管理的 `openai_hosted` 执行选择 Sandbox Provider、每个沙箱的资源以及不可变的 Runtime 发行版。PostgreSQL 为每个安装维护一个当前有效选择；Web 和 Core API 写入同一配置。节点文件保存其已安装副本和特定于主机的路径，且不能覆盖其资源或 Runtime。该选择独立于 Harness。部署可以保持未配置状态，没有节点；此时它拒绝托管准入。 @@ -50,7 +50,7 @@ POST 和 PUT 接受相同的完整选择，并要求提供先前 GET 返回的 ` | `cpus` | 整数，1 到 255 | | `memory_mib` | 整数，512 到 1048576 MiB | | `root_disk_mib` | microsandbox：至少 1024 MiB；Docker 和 E2B：省略或为零 | -| `environment_disk_mib` | microsandbox 自有磁盘：至少 1024 MiB；外部文件系统：零表示不请求配额；正值必须至少为 1024 MiB 且要求强制执行配额；Docker 和 E2B：省略或为零 | +| `environment_disk_mib` | microsandbox 自有磁盘：至少 1024 MiB；外部文件系统：每个新工作区的获准容量，零表示不请求配额，正值必须至少为 1024 MiB 且要求强制执行配额；Docker 和 E2B：省略或为零 | 这些限制描述每个沙箱。节点的 `max_active` 和 `max_retained` 是独立的预留限制，主机测量值绝不会允许超过其中任何一个。即使数值满足这些边界，原生提供商仍可能拒绝这些值。 @@ -61,7 +61,7 @@ microsandbox 会配置 CPU、内存、托管根磁盘，以及位于 `/environme 响应中可选的 `specification.workspace` 是从所选[工作区文件系统](../../../docs/zh/workspace-provider.md)派生的不可变能力回执，不是可独立写入的设置。它仅包含 `attachment`、`user_xattr` 和 `capacity_quota`；文件系统配置、路径和凭据不进入 Sandbox specification。Core 保存前校验组合声明，保留代次校验与节点构造均使用此回执。缺失 `workspace` 时保留自有磁盘模式及现有容量边界。Create 和 Resume 必须且仅当固定代次选择外部存储时提供绑定。 -为现有自有磁盘部署启用外部存储时，先选择文件系统配置，再使用兼容资源更新 Sandbox 部署（不支持配额的文件系统使用 `environment_disk_mib: 0`）。选择文件系统不会改变现有代次或 allocation。下一次 Sandbox 更新派生外部能力回执；旧的自有磁盘 allocation 保留原磁盘。文件系统选择必须支持 Provider 声明的要求；已采用外部存储的部署要求等价声明及兼容配额语义。 +为现有自有磁盘部署启用外部存储时，先选择文件系统配置，再使用兼容资源更新 Sandbox 部署（不支持配额的文件系统使用 `environment_disk_mib: 0`）。选择文件系统不会改变现有代次或 allocation。下一次 Sandbox 更新派生外部能力回执；旧的自有磁盘 allocation 保留原磁盘。文件系统选择必须支持 Provider 声明的要求。已采用外部存储的部署要求相同的 `attachment` 和 `user_xattr`；选择可以更改 `capacity_quota`，此后在 Sandbox 更新设置相匹配的 `environment_disk_mib` 之前拒绝新工作区。既有工作区保持其获准时的容量。 ### Runtime 发行版 {#runtime-release} diff --git a/docs/configuration.md b/docs/configuration.md index fb83fb72c..55ca7ceeb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -131,11 +131,60 @@ Select storage using `PUT /core/v1/workspace-storage` with the Core key as descr "root": "/srv/oac-workspaces", "namespace_id": "22222222-2222-4222-8222-222222222222", "uid": 65532 - } + }, + "tenant_capacity_mib": 0 } ``` -Then create or update the microsandbox deployment through the [sandbox deployment API](../contracts/agents-api/sandbox-deployment.md#routes), setting `resources.environment_disk_mib` to `0` and retaining the required compute resources, Runtime and provider configuration. A positive value requests a quota that this adapter does not enforce and is rejected. Deployment workspace requirements and each Session's attachment are derived from the selected database configuration; do not add a filesystem field to the deployment or a second storage configuration to node files. Web has no workspace storage editor. After the filesystem and deployment are selected, use **Nodes → Add node** to enroll the prepared hosts. Connected, ready compute alone does not qualify an external filesystem; validate actual Session creation and access through each participating node before admitting application traffic. Include the independent namespace in the [stopped-write backup and restore procedure](./getting-started/operations.md#back-up). +This configuration enforces no per-workspace quota, so `tenant_capacity_mib` must be `0`. For bounded workspaces, prepare the [quota mode storage server](#quota-mode-storage-server) and select a quota mode configuration with a [tenant capacity](#tenant-capacity) instead. + +Then create or update the microsandbox deployment through the [sandbox deployment API](../contracts/agents-api/sandbox-deployment.md#routes), retaining the required compute resources, Runtime and provider configuration. `resources.environment_disk_mib` is each new workspace's capacity: `0` without quota mode, where a positive value is rejected, and at least 1024 MiB in quota mode, where `0` is refused at workspace admission. Existing workspaces keep the capacity they were admitted with when the deployment changes. Deployment workspace requirements and each Session's attachment are derived from the selected database configuration; do not add a filesystem field to the deployment or a second storage configuration to node files. Web has no workspace storage editor. After the filesystem and deployment are selected, use **Nodes → Add node** to enroll the prepared hosts. Connected, ready compute alone does not qualify an external filesystem; validate actual Session creation and access through each participating node before admitting application traffic. Include the independent namespace in the [stopped-write backup and restore procedure](./getting-started/operations.md#back-up). + +#### Quota mode storage server + +Quota mode bounds each workspace's allocated bytes and inodes with an XFS project quota on the NFS server, set by the `oac-workspace-quota` command that Core calls over a restricted SSH forced command. [Quota mode](./workspace-provider.md#quota-mode) owns the protocol and enforcement semantics. Prepare the storage server as its administrator: + +1. Put the export on a local XFS filesystem mounted with enforcing project quotas (`prjquota`, not `pqnoenforce`). The export root must be a subdirectory of that mount, for example mount `/srv/xfs` and export `/srv/xfs/oac-workspaces`; the command keeps its journal in `/srv/xfs/.oac-quota`, outside the export. Prepare the namespace root and marker as above. +2. Install `xfsprogs` and Python 3.9 or newer. Install `storage/oac-workspace-quota` from the matching distribution as `/usr/local/sbin/oac-workspace-quota`, owned by root with mode `0755`. +3. Write `/etc/oac/workspace-quota.json`, owned by root and not writable by group or others: `{"export_root": "/srv/xfs/oac-workspaces", "mount_point": "/srv/xfs"}`. +4. Reserve a project ID range that no other project on this filesystem uses, then initialize the journal once: `oac-workspace-quota --config /etc/oac/workspace-quota.json init --projid-start 100000 --projid-end 199999`. It refuses to run over an existing journal; never initialize restored storage. +5. Generate a dedicated key pair, for example `ssh-keygen -t ed25519 -N '' -f oac-workspace-quota`. Authorize its public key for one account with a working shell, such as `oac-quota`, in that account's `authorized_keys`: `restrict,from="",command="sudo -n /usr/local/sbin/oac-workspace-quota --config /etc/oac/workspace-quota.json serve" ssh-ed25519 AAAA...`, with the sudoers entry `oac-quota ALL=(root) NOPASSWD: /usr/local/sbin/oac-workspace-quota --config /etc/oac/workspace-quota.json serve`. Alternatively authorize the key for root with the same options and the command without `sudo -n` under `PermitRootLogin forced-commands-only`. Keep `PermitUserEnvironment no`. +6. Read the server's public host key from `/etc/ssh/` on the server itself, not from an unauthenticated network scan. + +Select the quota mode configuration with the private key as its write-only credential. Delete the local private key file after a successful selection; Core keeps it encrypted with `secrets/core/credential.key`: + +```json +{ + "id": "33333333-3333-4333-8333-333333333333", + "adapter": "nfs", + "parameters": { + "root": "/srv/oac-workspaces", + "namespace_id": "22222222-2222-4222-8222-222222222222", + "uid": 65532, + "quota": { + "endpoint": "storage.internal:22", + "user": "oac-quota", + "host_key": "ssh-ed25519 AAAA..." + } + }, + "credential": { + "ssh_private_key": "-----BEGIN OPENSSH PRIVATE KEY-----\n...\n-----END OPENSSH PRIVATE KEY-----\n" + }, + "tenant_capacity_mib": 65536 +} +``` + +Selection calls the command's read-only check through the pinned host key. Then update the deployment with the per-workspace capacity in `resources.environment_disk_mib`. Until that update, an installation that switched from a configuration without quotas refuses new workspaces; existing workspaces continue with their own configuration. The credential belongs to the immutable configuration. To rotate it, authorize a new key, select a new configuration ID with it, and keep the old key authorized until no workspace bound to the old configuration is still being created. + +Supported recovery for quota mode is a block-level copy of the XFS filesystem; see [back up](./getting-started/operations.md#back-up). + +#### Tenant capacity + +`tenant_capacity_mib` is the installation's single tenant capacity policy, written only together with the selection by `PUT /core/v1/workspace-storage` and stored in Core's database. A positive value requires a quota mode configuration, and a quota mode configuration requires a positive value. It bounds the sum of admitted capacities of one tenant's workspaces that are not yet confirmed deleted, across configuration changes; [capability validation](./workspace-provider.md#capability-validation) owns the counting rules. A refused workspace makes Session execution unavailable; deleting Sessions frees capacity once their storage is removed. Lowering the value refuses new workspaces only. + +Workspaces created without quota mode remain unbounded; nothing resizes or migrates them. While one is retained, its tenant cannot admit new workspaces under a positive policy until that Session is deleted. + +The policy bounds what each tenant can commit, not the filesystem. Size the XFS filesystem for the number of tenants times the policy plus headroom for unbounded workspaces, identities, deletion markers, the journal and XFS metadata, or oversubscribe knowingly and monitor free space. Keep each node's `checkpoint_root` on separate storage. Capacity bounds provide no I/O fairness. ### Node capacity diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index c3e5b8ca7..d533e0a1f 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -128,6 +128,7 @@ A recoverable backup contains one consistent, stopped-write set. A database dump - The Docker volume `_data`, including `database/`, `secrets/` and `state/`. It holds Projects, key digests, nodes, default models, encrypted credentials and execution history, including large objects. Keep `secrets/core/credential.key` with its matching database or stored credentials cannot be decrypted. - The installation directory, including `.env`, `compose.yaml`, Compose overrides and the management command. - Every independent workspace namespace selected by current or retained objects, even when it is outside the installation volume. Back up its entire root, including `.oac-storage-root`, object identities, staging, live data, trash and deletion markers. Preserve numeric ownership, modes, links and extended attributes, including every `user.*` attribute. Copying only `live/data` loses lifecycle evidence and can resurrect deleted identities. [Workspace storage](../workspace-provider.md#kernel-nfs-adapter) owns the layout and ownership requirements. +- For a [quota mode](../workspace-provider.md#quota-mode) namespace, the entire XFS filesystem holding the export as one block-level copy: a storage or LVM snapshot, or an image of the frozen or unmounted device. Only such a copy keeps project IDs, quota limits, inode identities, the filesystem UUID and the `oac-workspace-quota` journal consistent. A file-level copy, `tar`, `rsync` or `xfsdump` loses them; the command then refuses the restored objects instead of serving them unbounded. - Each node's state directory, `/var/lib/oac-node/.oac/nodes//`, and its provider storage: Docker volumes or microsandbox's store. Include all compute and snapshot resources still referenced by the database. See [when a node host fails](./nodes.md#when-a-node-host-fails). - Each private `checkpoint_root` in its entirety, including `.oac-checkpoint-store`, archives, operation journals, lock files and tombstones, together with the matching Core database. It is separate from the native SDK store and independent workspace namespace. Preserve its identity and private ownership; the [checkpoint transfer contract](../sandbox-provider.md#checkpoint-transfer) owns its lifecycle. @@ -137,7 +138,7 @@ A recoverable backup contains one consistent, stopped-write set. A database dump 1. Block new application input, live file access and administrative mutations at the installation's ingress. Let active execution, initialization, file operations and native cleanup finish. Stop other applications or host processes that can write to the same filesystem. 2. While Core and nodes remain available, confirm that every owned writer has stopped and all Create, Kill, snapshot cleanup and filesystem mutations have settled. An offline node, timeout, missing instance or stopped service is insufficient. A confirmed suspension stops its source compute but retains a checkpoint that must be included in the backup. For the simplest cold recovery baseline, wait until eligible retained Sessions have completed checkpoint expiry cleanup and their allocations are released; retain their filesystem objects and native history under the [cold replacement contract](../sandbox-provider.md#cold-replacement-after-checkpoint-retention). 3. Stop Core and Web, then stop the node control services to prevent new lifecycle work. Recheck the exact native resources and keep all writers stopped until every part of the backup is complete. **Stopping Core or a node service does not stop its sandboxes.** The node service uses `KillMode=process`; microVMs and other provider-managed compute may continue to run independently. -4. Flush the stopped filesystem's pending writes using the storage platform's supported procedure and take its snapshot or export. Export the Core data volume and installation directory, plus required node state and provider stores, within the same window. Stop PostgreSQL before copying its `database/` files. Record the release, backup time, component inventory and checksums together, and keep the backup outside the installation being backed up. +4. Flush the stopped filesystem's pending writes using the storage platform's supported procedure and take its snapshot or export. For quota mode, freeze the XFS filesystem with `xfs_freeze -f` or unmount it while copying its device, then thaw it. Export the Core data volume and installation directory, plus required node state and provider stores, within the same window. Stop PostgreSQL before copying its `database/` files. Record the release, backup time, component inventory and checksums together, and keep the backup outside the installation being backed up. 5. Restore service availability only after all copies finish. Bring up storage before Core and nodes, verify readiness, then reopen ingress. Do not replay uncertain native work to make a backup pass. There is no one-command, lossless maintenance drain. Archive and deployment reset change Session lifecycle state; they are not substitutes for a backup pause that preserves resumability. If any writer or mutation remains unknown, keep the backup incomplete and resolve that ownership first. @@ -155,7 +156,7 @@ A logical dump still needs the matching secrets, state, independent filesystems Keep application ingress and Core/node services stopped while restoring. Fence or shut down every old writer before exposing the restored filesystem; an unreachable old host is not a fence. A rehearsal must use an isolated installation and storage copy that cannot connect to the original nodes or share their writable namespace. -Restore matching secrets and database state, the complete independent filesystem namespaces and any required node identities/provider stores before starting Core. Preserve the database's configuration and namespace UUIDs, object identities and deletion markers; do not run first-time namespace initialization over restored storage. Reestablish the configured mount paths and service UIDs, then validate the mounted filesystem, ownership and xattrs from each participating service context. [Independent workspace storage](../configuration.md#independent-workspace-storage) owns deployment setup. +Restore matching secrets and database state, the complete independent filesystem namespaces and any required node identities/provider stores before starting Core. Preserve the database's configuration and namespace UUIDs, object identities and deletion markers; do not run first-time namespace initialization over restored storage. Restore a quota mode filesystem by writing its block-level copy to the backing device and mounting it with `prjquota` at the configured mount point; never run `oac-workspace-quota init` over it, and do not mount the copy beside the original, which has the same filesystem UUID. Reestablish the configured mount paths and service UIDs, then validate the mounted filesystem, ownership and xattrs from each participating service context. [Independent workspace storage](../configuration.md#independent-workspace-storage) owns deployment setup. The cold released baseline resumes only qualified native history after new compute admission. A backup containing retained checkpoints also depends on the matching native runtime, node identity, provider store and external filesystem identities. File-level restore can change inode identity and prevent strict checkpoint restore even when file bytes match. This procedure does not promise portable memory snapshots or restoration onto an arbitrary replacement node; preserve unresolved ownership rather than substituting a new sandbox. diff --git a/docs/workspace-provider.md b/docs/workspace-provider.md index 9900d0d26..37d287c13 100644 --- a/docs/workspace-provider.md +++ b/docs/workspace-provider.md @@ -8,7 +8,9 @@ The independent workspace filesystem boundary is defined in [`workspacefs.go`](h `Reference` contains immutable, canonical lowercase, nonzero UUIDs for `TenantID`, `EnvironmentID` and `ObjectID`. Core persists the reference before creation. Object identities are never reused, including after deletion or an unconfirmed mutation. An adapter must verify the entire ownership tuple before exposing or deleting an object. -`Configuration` is an immutable Core database record with its own canonical UUID `ID`, an `Adapter` identifier and adapter-owned JSON object `Parameters`. The database is its only authored home; changing configuration creates a new identity. Core forwards parameters without interpreting native paths. Existing objects remain bound to their original configuration. +`Configuration` is an immutable Core database record with its own canonical UUID `ID`, an `Adapter` identifier and adapter-owned JSON object `Parameters`. The database is its only authored home; changing configuration creates a new identity. Core forwards parameters without interpreting native paths. Existing objects remain bound to their original configuration. Parameters are public: they reach nodes and administrator responses. + +`ConfigurationRecord` adds the configuration's adapter-encoded private `Credential`, empty when the parameters require none. The credential is part of the immutable configuration. Core seals it with the installation's credential encryption key, bound to the installation ID and configuration ID, and opens it only to construct `Control` in Core. It never appears in a `Binding`, an administrator response, an audit record or a log, and nodes never receive it. `Attachment` binds an adapter-issued JSON object `Native` to the complete reference, configuration ID and attachment kind. It is internal ownership evidence, not an application-provided host path. Shared validation checks the envelope and expected reference/configuration match; the selected adapter must also validate its parameters, receipt and native ownership. Structural validity alone never authorizes filesystem access. Public Session payloads must not supply attachments, configuration parameters or resolved host directories. @@ -16,13 +18,15 @@ The independent workspace filesystem boundary is defined in [`workspacefs.go`](h ## Operations and outcomes +An adapter registers one implementation of `Adapter` with three separate, mandatory roles. `Normalize(parameters, credential)` validates and canonicalizes parameters and encodes the credential input without I/O; a missing required credential or a credential the parameters do not use is `ErrInvalid`, and an encoded credential is valid input that encodes to itself, so Core can re-select a retained configuration with its stored credential. `Control(ConfigurationRecord)` constructs the control adapter in Core with the decrypted credential. `Resolver(Configuration)` constructs the node resolver from public configuration only. Construction performs no filesystem or network operation. + Every control adapter implements `Declaration`, `Check`, `Create`, `Observe` and `Delete`. Every node attachment adapter implements `Resolve`. There is no optional side interface, implicit adapter substitution or nil implementation fallback. | Operation | Required meaning | | --- | --- | | `Declaration()` | Describe the selected configuration's verified attachment, user xattr and quota behavior. | | `Check(ctx)` | Validate configuration and availability without creating an object. | -| `Create(ctx, reference)` | Create the owned object or converge on its existing attachment without overwriting data; retries and concurrent calls use the same immutable reference. | +| `Create(ctx, reference, capacityMiB)` | Create the owned object or converge on its existing attachment without overwriting data; retries and concurrent calls use the same immutable reference and admitted capacity. A positive capacity is enforced, and verified on every retry, before success. | | `Observe(ctx, reference)` | Return an existing object's bound attachment without creating, repairing or preparing it. | | `Delete(ctx, reference)` | Converge on terminal deletion of the verified owned object after Core has authorized deletion and confirmed compute stop. | | `Resolve(ctx, binding)` | Validate native ownership and resolve the attachment for local compute. | @@ -35,7 +39,11 @@ Same-reference retry and concurrent convergence are mandatory adapter guarantees `Requirements` declares the compute placement's attachment kind and need for `user_xattr`. `Declaration` reports the filesystem adapter's attachment kind, `UserXAttr` support and `CapacityQuota` enforcement. `ValidateCombination` is the shared pre-allocation admission check. The current attachment kind is `host_directory`; unknown kinds are rejected. A required user xattr capability means the mounted filesystem supports user extended attributes used by Environment preparation. -A zero requested capacity means no quota was requested. A positive capacity in MiB is admitted only when the adapter actually enforces that limit; reporting available space or accepting an unchecked number is insufficient. The adapter's immutable configuration must enforce the admitted bound. Adapters cannot claim quota support based only on the Sandbox Provider's compute disk size. +A zero requested capacity means no quota was requested. A positive capacity in MiB is admitted only when the adapter actually enforces that limit; reporting available space or accepting an unchecked number is insufficient. `CapacityQuota` means the adapter bounds every object created with a positive capacity: the object's allocated bytes, including directory and metadata blocks, and its inodes, not the logical size of its files. Writes beyond the bound fail with an error on write or fsync; nothing is truncated. Adapters cannot claim quota support based only on the Sandbox Provider's compute disk size. No I/O fairness follows from a capacity bound. + +Capacity is admitted per object. For a new binding, Core validates the selected configuration's declaration against the deployment's request before binding; the binding transaction then refuses with a retryable conflict if another configuration was selected meanwhile. Core persists the admitted capacity with the reference and passes it unchanged to every `Create`. A retained binding is validated against its own configuration and admitted capacity, so changing the deployment's request or the selection never resizes or rebinds existing storage. + +When the [tenant capacity policy](configuration.md#tenant-capacity) is set, a new binding is also admitted against the tenant's committed capacity: the sum of admitted capacities of that tenant's workspaces that are not yet confirmed deleted, across all configurations. Creating, failed or unconfirmed and deleting workspaces count until the adapter confirms deletion. A retained workspace with no admitted capacity counts as unbounded and blocks further admission for its tenant until it is deleted. Under the policy a request without capacity is unsupported, and a request that would exceed it fails as unavailable before any storage or compute is allocated. Existing bindings never re-enter admission, and lowering the policy evicts nothing. Workspace access is single-writer. Core must serialize compute ownership and confirm the previous writer stopped before attaching a replacement. There is no cross-VM lock capability in this protocol: a guest `flock` is not proof of a host lock or another guest's exclusion. A placement must not advertise concurrent-safe attachment based on guest-local locking. @@ -49,13 +57,23 @@ Workspace storage is retained with its Session until explicit Session deletion. The `nfs` adapter uses an operator-premounted Linux kernel NFS filesystem. The supported setup is NFSv4.2 on trusted AUTH_SYS clients with `root_squash`, using the same export and absolute mount path on Core and nodes. The adapter neither mounts exports nor manages NFS credentials or service accounts. Other operating systems return `ErrUnsupported`. [Configuration](./configuration.md#independent-workspace-storage) owns installation steps and selection. -Parameters contain exactly `root`, `namespace_id` and `uid`. `root` is a clean absolute mount path; `namespace_id` is a canonical nonzero UUID; `uid` is the nonzero effective host UID required on both Core and nodes. The initial deployment uses 65532. The operator prepares the mounted root with mode `0700`, owned by this UID, and a regular `.oac-storage-root` file with mode `0600`, owned by the same UID, containing exactly the namespace UUID followed by one newline. Do this on the export's server-side storage as its administrator; root-squashed clients cannot assume privileged ownership changes. The root marker is an assertion of the selected export's identity, not a credential or distributed lock. +Parameters contain `root`, `namespace_id`, `uid` and, in quota mode, `quota`. `root` is a clean absolute mount path; `namespace_id` is a canonical nonzero UUID; `uid` is the nonzero effective host UID required on both Core and nodes. The initial deployment uses 65532. The operator prepares the mounted root with mode `0700`, owned by this UID, and a regular `.oac-storage-root` file with mode `0600`, owned by the same UID, containing exactly the namespace UUID followed by one newline. Do this on the export's server-side storage as its administrator; root-squashed clients cannot assume privileged ownership changes. The root marker is an assertion of the selected export's identity, not a credential or distributed lock. Construction validates and canonicalizes JSON without filesystem I/O. `Check` verifies the opened root is kernel NFS, the effective UID and private ownership/modes match, the namespace marker matches, and real file-descriptor `user.*` xattr set/get/remove operations succeed. Every filesystem operation, including opening the root and checking availability, runs within one process-wide budget of 32 in-flight operations. A timed-out caller receives `ErrUnconfirmed`; its slot remains occupied until the kernel call returns. An exhausted budget returns `ErrUnavailable`. Directory-descriptor-relative operations stay anchored to the opened root. An unavailable mount or a local directory at the configured path fails validation; no local fallback is created. -The declaration is `host_directory`, `user_xattr: true`, `capacity_quota: false`. The native attachment contains only the namespace identity; resolution validates the complete binding against the immutable configuration and filesystem ownership before returning `objects//live/data` beneath the root. Only this data directory is exposed to the guest. Neither tenant nor application input supplies a host path. +The declaration is `host_directory`, `user_xattr: true` and `capacity_quota` true exactly in quota mode. The native attachment contains only the namespace identity; resolution validates the complete binding against the immutable configuration and filesystem ownership before returning `objects//live/data` beneath the root. Only this data directory is exposed to the guest. Neither tenant nor application input supplies a host path. + +Without `quota`, the adapter enforces no per-Environment byte or inode quota and requires no credential. One object can exhaust the shared filesystem, preventing writes to other objects and the durable terminal metadata required for safe deletion. Operators must monitor free bytes and inodes and maintain headroom for lifecycle metadata and cleanup. + +### Quota mode + +Quota mode bounds each object with an XFS project quota enforced by the NFS server's own filesystem. `quota` contains exactly `endpoint` (`host:port` of the storage server's SSH service), `user` (the restricted account) and `host_key` (the server's pinned OpenSSH public host key in authorized-key form; a comment is dropped). The credential is `{"ssh_private_key": "..."}`, one unencrypted OpenSSH private key whose public key the server authorizes only for the `oac-workspace-quota` forced command. [Configuration](configuration.md#quota-mode-storage-server) owns server preparation. + +For a positive capacity, `Create` calls the command after claiming the identity and creating the object's empty `staging/` and `trash/`, and before creating a stage or returning an existing `live` object, so every retry re-verifies the bound. The request carries the complete reference, configuration ID, namespace ID, capacity and an inode limit of 256 per MiB; it can name no path, project ID, user or command. The client verifies the pinned host key, keeps the key in memory and runs one bounded session per call. A failure before the request is sent is `ErrUnavailable`; a missing or malformed answer afterwards is `ErrUnconfirmed`, and the same reference converges on retry. `Check` also calls the command, proving reachability, the host key, the namespace marker, the journal and enforcement before selection. + +The command runs as root on the storage server. Holding one lock, it opens `objects//staging` relative to the configured export without following links, verifies the service UID, private modes, device and the full identity, refuses a deleted object and refuses a project on `objects/` or the object directory. A local SQLite journal on the same XFS filesystem, outside the export, records each object's reference, configuration, capacity, inode limit, staging inode and generation, and a project ID allocated from an operator-reserved range and never reused. The allocation commits before any change; one `FS_IOC_FSSETXATTR` then sets the project ID and inheritance on the still-empty staging directory, and the server sets bytes and inodes hard limits with zero soft limits and reads them back. Stages inherit the project; `live` and retired trash keep it through rename. The identity, deletion marker, object directory and `trash/` parent stay outside the project, so a full object can still be marked deleted and removed. Removal releases the data allocation; the retained `staging/` inode remains charged to the project. A journal for another filesystem or namespace, disabled enforcement, an exhausted range, existing user data, a foreign or inherited project, a replaced staging inode, a changed reference, configuration or capacity, and changed or lost limits on a ready object are refused rather than repaired. A project without a dquot reads as zero limits, never as an unused ID. -The NFS adapter enforces no per-Environment or per-tenant byte or inode quota. One object can exhaust the shared filesystem, preventing writes to other objects and the durable terminal metadata required for safe deletion. Operators must monitor free bytes and inodes and maintain headroom for lifecycle metadata and cleanup. A capacity bound on the whole namespace does not isolate capacity between tenants or objects. +The bound is authoritative when `Create` succeeds; `Resolve` does not re-attest it, so resuming a ready workspace does not depend on the storage server's SSH service. A storage administrator who later disables quotas or clears limits defeats enforcement until the supported procedure restores them. Project IDs and limits live in the XFS filesystem, so quota mode requires the [block-level backup](getting-started/operations.md#back-up) described for it. Each object owns `objects//identity`, `staging/`, `live/`, `trash/` and, after deletion begins, `deleted-marker`. The permanent identity binds the tenant, Environment and object UUIDs. Create prepares and syncs a private unique staging envelope before atomically publishing its nonempty directory as `live`; replay cannot overwrite existing live data. Delete first records a durable object-local terminal marker, retires live data to unique object-local trash and removes data before its ownership markers. Concurrent deleters converge. Minimal identity and deletion metadata remain intentionally; deletion does not mean every metadata file disappears. Create and Resolve refuse the retired identity. A late in-flight Create can leave empty controlled metadata requiring a repeated Delete after its syscall settles; it cannot authorize a new writer or reuse the object identity. Cleanup examines only that object's staging and trash, without a namespace-wide sweep or background service. diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 7cf303515..cd5792b34 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: 6bfcaa201c332aa1907f6ab7b0e0ffde4c8f635b363dbea2d0633947615e7264 +source_hash: 2d0f6969acaa9383ef74c639547219e3146366f91a0ab5d42ad734569533c144 --- Core 安装的每项设置都恰好只有一个归属位置，分属以下三类： @@ -135,11 +135,60 @@ services: "root": "/srv/oac-workspaces", "namespace_id": "22222222-2222-4222-8222-222222222222", "uid": 65532 - } + }, + "tenant_capacity_mib": 0 } ``` -随后通过[沙箱部署 API](../../contracts/agents-api/zh/sandbox-deployment.md#routes)创建或更新 microsandbox 部署，将 `resources.environment_disk_mib` 设为 `0`，并保留所需计算资源、Runtime 和 Provider 配置。正数表示请求配额，本适配器不实施此配额，因此会拒绝。部署的工作区要求和每个 Session 的挂载凭据均从已选数据库配置派生；不要向部署添加文件系统字段，也不要在节点文件中添加第二份存储配置。Web 没有工作区存储编辑器。选择文件系统和部署后，通过 **Nodes → Add node** 注册已准备的主机。计算资源已连接且就绪并不足以证明外部文件系统具备资格；准入应用流量前，通过每个参与节点验证实际 Session 创建和访问。将独立命名空间纳入[停止写入后的备份恢复流程](./getting-started/operations.md#back-up)。 +此配置不实施单个工作区配额，因此 `tenant_capacity_mib` 必须为 `0`。如需有上限的工作区，应准备[配额模式存储服务器](#quota-mode-storage-server)，并改为选择带[租户容量](#tenant-capacity)的配额模式配置。 + +随后通过[沙箱部署 API](../../contracts/agents-api/zh/sandbox-deployment.md#routes)创建或更新 microsandbox 部署，并保留所需计算资源、Runtime 和 Provider 配置。`resources.environment_disk_mib` 是每个新工作区的容量：非配额模式下为 `0`，正数会被拒绝；配额模式下至少为 1024 MiB，`0` 会在工作区准入时被拒绝。部署变更时，既有工作区保持其获准时的容量。部署的工作区要求和每个 Session 的挂载凭据均从已选数据库配置派生；不要向部署添加文件系统字段，也不要在节点文件中添加第二份存储配置。Web 没有工作区存储编辑器。选择文件系统和部署后，通过 **Nodes → Add node** 注册已准备的主机。计算资源已连接且就绪并不足以证明外部文件系统具备资格；准入应用流量前，通过每个参与节点验证实际 Session 创建和访问。将独立命名空间纳入[停止写入后的备份恢复流程](./getting-started/operations.md#back-up)。 + +#### 配额模式存储服务器 {#quota-mode-storage-server} + +配额模式通过 NFS 服务器上的 XFS project quota 为每个工作区的已分配字节和 inode 设定上限，由 Core 经受限 SSH 强制命令调用的 `oac-workspace-quota` 命令设置。协议和强制执行语义由[配额模式](./workspace-provider.md#quota-mode)负责。以存储服务器管理员身份完成以下准备： + +1. 将导出放在启用强制 project quota（`prjquota`，而非 `pqnoenforce`）挂载的本地 XFS 文件系统上。导出根目录必须是该挂载点的子目录，例如挂载 `/srv/xfs`、导出 `/srv/xfs/oac-workspaces`；该命令将日志库保存在导出之外的 `/srv/xfs/.oac-quota`。按上文准备命名空间根目录和标记。 +2. 安装 `xfsprogs` 和 Python 3.9 或更高版本。从匹配的发行包将 `storage/oac-workspace-quota` 安装为 `/usr/local/sbin/oac-workspace-quota`，所有者为 root，权限 `0755`。 +3. 写入 `/etc/oac/workspace-quota.json`，所有者为 root，组和其他用户不可写：`{"export_root": "/srv/xfs/oac-workspaces", "mount_point": "/srv/xfs"}`。 +4. 预留该文件系统上其他 project 均未使用的 project ID 范围，然后仅初始化一次日志库：`oac-workspace-quota --config /etc/oac/workspace-quota.json init --projid-start 100000 --projid-end 199999`。它拒绝在已有日志库上运行；绝不要初始化恢复后的存储。 +5. 生成专用密钥对，例如 `ssh-keygen -t ed25519 -N '' -f oac-workspace-quota`。在一个具有可用 shell 的账户（如 `oac-quota`）的 `authorized_keys` 中授权其公钥：`restrict,from="",command="sudo -n /usr/local/sbin/oac-workspace-quota --config /etc/oac/workspace-quota.json serve" ssh-ed25519 AAAA...`，并添加 sudoers 条目 `oac-quota ALL=(root) NOPASSWD: /usr/local/sbin/oac-workspace-quota --config /etc/oac/workspace-quota.json serve`。也可以在 `PermitRootLogin forced-commands-only` 下为 root 使用相同选项授权该密钥，命令中不带 `sudo -n`。保持 `PermitUserEnvironment no`。 +6. 在服务器本机的 `/etc/ssh/` 中读取其公共主机密钥，不要通过未经认证的网络扫描获取。 + +选择配额模式配置时，将私钥作为只写凭据提交。选择成功后删除本地私钥文件；Core 使用 `secrets/core/credential.key` 加密保存它： + +```json +{ + "id": "33333333-3333-4333-8333-333333333333", + "adapter": "nfs", + "parameters": { + "root": "/srv/oac-workspaces", + "namespace_id": "22222222-2222-4222-8222-222222222222", + "uid": 65532, + "quota": { + "endpoint": "storage.internal:22", + "user": "oac-quota", + "host_key": "ssh-ed25519 AAAA..." + } + }, + "credential": { + "ssh_private_key": "-----BEGIN OPENSSH PRIVATE KEY-----\n...\n-----END OPENSSH PRIVATE KEY-----\n" + }, + "tenant_capacity_mib": 65536 +} +``` + +选择操作会通过固定的主机密钥调用该命令的只读检查。随后在 `resources.environment_disk_mib` 中设置每个工作区的容量并更新部署。在此更新之前，从非配额配置切换过来的安装会拒绝新工作区；既有工作区继续使用其自身配置。凭据属于不可变配置。如需轮换，先授权新密钥，使用新密钥选择新的配置 ID，并在仍有绑定旧配置的工作区处于创建过程中时保留旧密钥的授权。 + +配额模式支持的恢复方式是对 XFS 文件系统进行块级复制；参见[备份](./getting-started/operations.md#back-up)。 + +#### 租户容量 {#tenant-capacity} + +`tenant_capacity_mib` 是安装唯一的租户容量策略，只能与选择一起通过 `PUT /core/v1/workspace-storage` 写入，并保存在 Core 数据库中。正值要求配额模式配置，配额模式配置也要求正值。它限制单个租户所有尚未确认删除的工作区的获准容量之和，并跨配置变更生效；计数规则由[能力验证](./workspace-provider.md#capability-validation)负责。被拒绝的工作区使 Session 执行不可用；删除 Session 后，其存储被移除时即释放容量。调低该值只拒绝新工作区。 + +非配额模式下创建的工作区保持无上限；不会调整其大小或迁移它们。只要保留此类工作区，在正策略下其租户在删除该 Session 前无法准入新工作区。 + +该策略限制每个租户可承诺的容量，而不限制文件系统。应按租户数乘以策略值，并为无上限工作区、标识、删除标记、日志库和 XFS 元数据预留余量来规划 XFS 文件系统大小；或者明知超额分配并监测可用空间。每个节点的 `checkpoint_root` 应放在独立存储上。容量上限不提供 I/O 公平性。 ### 节点容量 {#node-capacity} diff --git a/docs/zh/getting-started/operations.md b/docs/zh/getting-started/operations.md index 4316ac34d..260d073ec 100644 --- a/docs/zh/getting-started/operations.md +++ b/docs/zh/getting-started/operations.md @@ -1,7 +1,7 @@ --- title: "运维" source: docs/getting-started/operations.md -source_hash: 591196d61ed9946435d07afcbbcc2f53d64b9db0cf6b4806ddfb0f85a16974ba +source_hash: 8ce67b66748a8414ac3ee954b07e379b7d3a0a37790bb162c000c9ae556842fc --- 安装运维人员负责 Core 主机、存储和可用性。节点主机运行各自的服务；参阅[节点](nodes.md)。设置见[配置参考](../configuration.md)。 @@ -130,6 +130,7 @@ Core 记录每次公开资源写入所使用的密钥；历史保留策略为 [` - Docker 卷 `_data`，包括 `database/`、`secrets/` 和 `state/`。其中保存 Project、密钥摘要、节点、默认模型、加密凭据和执行历史（含大对象）。`secrets/core/credential.key` 必须与匹配的数据库一起保留，否则存储的凭据无法解密。 - 安装目录，包括 `.env`、`compose.yaml`、Compose 覆盖文件和管理命令。 - 当前或保留对象所使用的每个独立工作区命名空间，即使其位于安装卷之外。备份整个根目录，包括 `.oac-storage-root`、对象标识、staging、live 数据、trash 和删除标记。保留数值所有者、权限、链接及扩展属性，包括全部 `user.*` 属性。仅复制 `live/data` 会丢失生命周期证据，并可能使已删除标识复活。[工作区存储](../workspace-provider.md#kernel-nfs-adapter)负责定义布局与所有权要求。 +- 对于[配额模式](../workspace-provider.md#quota-mode)命名空间，将承载导出的整个 XFS 文件系统作为一份块级副本备份：存储或 LVM 快照，或已冻结或已卸载设备的镜像。只有这样的副本能保持 project ID、配额限制、inode 标识、文件系统 UUID 和 `oac-workspace-quota` 日志库的一致。文件级复制、`tar`、`rsync` 或 `xfsdump` 会丢失这些信息；此时该命令会拒绝恢复后的对象，而不是在无上限状态下提供它们。 - 各节点状态目录 `/var/lib/oac-node/.oac/nodes//` 及其 Provider 存储：Docker 卷或 microsandbox 存储。包含数据库仍然引用的全部计算资源和快照。参见[节点主机故障时](./nodes.md#when-a-node-host-fails)。 - 每个私有 `checkpoint_root` 的全部内容，包括 `.oac-checkpoint-store`、归档、operation journal、lock 文件与 tombstone，并与匹配的 Core 数据库一起保留。它独立于原生 SDK store 和工作区 namespace。保留其身份和私有所有权；[检查点转移契约](../sandbox-provider.md#checkpoint-transfer)定义其生命周期。 @@ -139,7 +140,7 @@ Core 记录每次公开资源写入所使用的密钥；历史保留策略为 [` 1. 在安装入口阻止新应用 input、实时文件访问和管理变更。等待活动执行、初始化、文件操作和原生清理完成。停止其他能够写入同一文件系统的应用或主机进程。 2. 保持 Core 和节点可用，确认每个所属写入方均已停止，且所有 Create、Kill、快照清理及文件系统变更均已结算。节点离线、超时、实例缺失或服务停止均不足以证明这一点。确认挂起会停止源计算资源，但保留的检查点也必须纳入备份。最简单的冷恢复基线是等待符合条件的保留 Session 完成检查点到期清理、allocation 已释放，再按[冷替换契约](../sandbox-provider.md#cold-replacement-after-checkpoint-retention)保留文件系统对象及原生历史。 3. 停止 Core 和 Web，再停止节点控制服务，以防止新的生命周期操作。重新核实精确的原生资源，保持所有写入方停止，直到备份各部分全部完成。**停止 Core 或节点服务不会停止其沙箱。** 节点服务使用 `KillMode=process`；microVM 及其他由 Provider 管理的计算资源可能继续独立运行。 -4. 使用存储平台支持的流程将已停止写入的文件系统中的待写数据落盘，并创建其快照或导出。在同一窗口内导出 Core 数据卷、安装目录及所需的节点状态和 Provider 存储。复制 PostgreSQL 的 `database/` 文件前必须停止 PostgreSQL。一起记录版本、备份时间、组成清单和校验值，并将备份保存在被备份安装之外。 +4. 使用存储平台支持的流程将已停止写入的文件系统中的待写数据落盘，并创建其快照或导出。对于配额模式，在复制其设备期间使用 `xfs_freeze -f` 冻结 XFS 文件系统或将其卸载，完成后解冻。在同一窗口内导出 Core 数据卷、安装目录及所需的节点状态和 Provider 存储。复制 PostgreSQL 的 `database/` 文件前必须停止 PostgreSQL。一起记录版本、备份时间、组成清单和校验值，并将备份保存在被备份安装之外。 5. 所有复制完成后才恢复服务。先恢复存储，再启动 Core 和节点，验证就绪后开放入口。不要通过重放不确定的原生操作来让备份通过。 当前没有一条命令即可完成且无损的维护排空流程。归档和部署重置会改变 Session 生命周期状态，不能替代保持续接能力的备份暂停。任何写入方或变更仍然未知时，应将备份保留为未完成状态，先解决相应归属。 @@ -157,7 +158,7 @@ docker compose -f "$HOME/.oac/core/compose.yaml" exec -T database \ 恢复期间保持应用入口及 Core/节点服务关闭。暴露恢复后的文件系统前，必须隔离或关闭每一个旧写入方；旧主机不可达不代表已隔离。演练必须使用隔离的安装和存储副本，不能连接原节点，也不能共享其可写命名空间。 -启动 Core 前，恢复相匹配的 secrets 和数据库状态、完整的独立文件系统命名空间，以及任何所需的节点标识和 Provider 存储。保留数据库中的配置和命名空间 UUID、对象标识及删除标记；不要对恢复后的存储执行首次命名空间初始化。重新建立配置中的挂载路径和服务 UID，然后从每个参与服务的上下文验证所挂载文件系统、所有权及 xattr。[独立工作区存储](../configuration.md#independent-workspace-storage)负责定义部署设置。 +启动 Core 前，恢复相匹配的 secrets 和数据库状态、完整的独立文件系统命名空间，以及任何所需的节点标识和 Provider 存储。保留数据库中的配置和命名空间 UUID、对象标识及删除标记；不要对恢复后的存储执行首次命名空间初始化。恢复配额模式文件系统时，将其块级副本写入后备设备，并以 `prjquota` 挂载到配置的挂载点；绝不要对其运行 `oac-workspace-quota init`，也不要将副本与原文件系统同时挂载，二者具有相同的文件系统 UUID。重新建立配置中的挂载路径和服务 UID，然后从每个参与服务的上下文验证所挂载文件系统、所有权及 xattr。[独立工作区存储](../configuration.md#independent-workspace-storage)负责定义部署设置。 已释放计算资源的冷恢复基线仅在新计算资源准入后续接具备资格的原生历史。包含保留检查点的备份还依赖匹配的原生 Runtime、节点标识、Provider 存储和外部文件系统标识。文件级恢复可能改变 inode 标识，即使文件字节一致，也可能阻止严格检查点恢复。此流程不承诺可移植内存快照，也不承诺恢复到任意替代节点；应保留未解决的归属，而不是替换为新沙箱。 diff --git a/docs/zh/workspace-provider.md b/docs/zh/workspace-provider.md index 8c681069a..e5383a734 100644 --- a/docs/zh/workspace-provider.md +++ b/docs/zh/workspace-provider.md @@ -1,7 +1,7 @@ --- title: 工作区文件系统 Provider source: docs/workspace-provider.md -source_hash: 21eb220ac59a477a8e799a6de20e01b339536901e8c9685695d2ac498b1b7bdb +source_hash: ff018b1117a7f87786a23737212f1ed6b265d17fde43a0ef847f9f104788df2c --- 独立工作区文件系统边界由 [`workspacefs.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/workspacefs/workspacefs.go) 定义。本文规定必需的集成契约，并不表示所有 Sandbox Provider 或执行位置均已实现。文件系统适配器拥有存储对象与原生挂载解析职责；[Sandbox Provider](sandbox-provider.md) 拥有计算资源职责。Core 在分配任一资源前选择并验证二者的组合。 @@ -10,7 +10,9 @@ source_hash: 21eb220ac59a477a8e799a6de20e01b339536901e8c9685695d2ac498b1b7bdb `Reference` 包含 `TenantID`、`EnvironmentID` 和 `ObjectID`，均为不可变、规范小写且非零的 UUID。Core 在创建前持久化引用。对象标识永不复用，包括删除后或变更结果尚未确认时。适配器必须在暴露或删除对象前验证完整的归属元组。 -`Configuration` 是 Core 数据库中的不可变记录，具有规范 UUID `ID`、`Adapter` 标识和由适配器定义的 JSON 对象 `Parameters`。数据库是其唯一编写来源；更改配置须创建新的标识。Core 转发参数而不解释原生路径。现有对象始终绑定其原始配置。 +`Configuration` 是 Core 数据库中的不可变记录，具有规范 UUID `ID`、`Adapter` 标识和由适配器定义的 JSON 对象 `Parameters`。数据库是其唯一编写来源；更改配置须创建新的标识。Core 转发参数而不解释原生路径。现有对象始终绑定其原始配置。参数是公开的：会发送到节点并出现在管理员响应中。 + +`ConfigurationRecord` 在配置之上增加由适配器编码的私有 `Credential`；参数不需要凭据时为空。凭据属于不可变配置。Core 使用安装的凭据加密密钥对其加密，并绑定安装 ID 与配置 ID，仅在 Core 中构造 `Control` 时解密。凭据绝不出现在 `Binding`、管理员响应、审计记录或日志中，节点也绝不会收到它。 `Attachment` 将适配器签发的 JSON 对象 `Native` 绑定到完整引用、配置 ID 和挂载类型。它是内部归属凭证，不是应用提供的主机路径。共享验证检查信封结构以及预期引用和配置的匹配；所选适配器还必须验证参数、凭证及原生归属。结构有效本身绝不授权文件系统访问。公开 Session 载荷不得提供挂载凭证、配置参数或解析后的主机目录。 @@ -18,13 +20,15 @@ source_hash: 21eb220ac59a477a8e799a6de20e01b339536901e8c9685695d2ac498b1b7bdb ## 操作与结果 {#operations-and-outcomes} +适配器注册一个 `Adapter` 实现，其三个角色相互独立且均为必需。`Normalize(parameters, credential)` 在不执行 I/O 的情况下验证并规范化参数、编码凭据输入；缺少必需凭据或提供参数不使用的凭据均为 `ErrInvalid`；编码后的凭据是有效输入且编码结果不变，因此 Core 可以使用已存储的凭据重新选择保留的配置。`Control(ConfigurationRecord)` 在 Core 中使用解密后的凭据构造控制适配器。`Resolver(Configuration)` 仅根据公开配置构造节点解析器。构造过程不执行文件系统或网络操作。 + 每个控制适配器实现 `Declaration`、`Check`、`Create`、`Observe` 和 `Delete`；每个节点挂载适配器实现 `Resolve`。不得使用可选旁路接口、隐式替换适配器或 nil 实现回退。 | 操作 | 必需语义 | | --- | --- | | `Declaration()` | 描述所选配置已经验证的挂载、用户扩展属性和配额行为。 | | `Check(ctx)` | 验证配置和可用性，不创建对象。 | -| `Create(ctx, reference)` | 创建归属对象或收敛到其现有挂载凭证，不覆盖数据；重试与并发调用使用同一不可变引用。 | +| `Create(ctx, reference, capacityMiB)` | 创建归属对象或收敛到其现有挂载凭证，不覆盖数据；重试与并发调用使用同一不可变引用和获准容量。正容量须在成功返回前实际强制执行，并在每次重试时重新验证。 | | `Observe(ctx, reference)` | 返回现有对象的绑定挂载凭证，不创建、修复或准备对象。 | | `Delete(ctx, reference)` | Core 授权删除且确认计算资源停止后，收敛到已验证归属对象的终态删除。 | | `Resolve(ctx, binding)` | 验证原生归属并为本地计算解析挂载。 | @@ -37,7 +41,11 @@ source_hash: 21eb220ac59a477a8e799a6de20e01b339536901e8c9685695d2ac498b1b7bdb `Requirements` 声明计算位置的挂载类型及 `user_xattr` 需求。`Declaration` 报告文件系统适配器的挂载类型、`UserXAttr` 支持和 `CapacityQuota` 强制执行能力。`ValidateCombination` 是共享的分配前准入检查。当前挂载类型为 `host_directory`；未知类型会被拒绝。用户扩展属性能力表示挂载后的文件系统支持 Environment 准备流程使用的用户扩展属性。 -请求容量为零表示未请求配额。以 MiB 为单位的正容量仅在适配器实际强制执行该限制时获准；报告可用空间或接受未经检查的数字均不足够。适配器的不可变配置必须强制执行获准上限。适配器不能仅根据 Sandbox Provider 的计算磁盘大小声明配额支持。 +请求容量为零表示未请求配额。以 MiB 为单位的正容量仅在适配器实际强制执行该限制时获准；报告可用空间或接受未经检查的数字均不足够。`CapacityQuota` 表示适配器为每个以正容量创建的对象设定上限：限制对象已分配的字节（包括目录和元数据块）及其 inode 数量，而非文件的逻辑大小。超出上限的写入在 write 或 fsync 时报错，不会截断任何内容。适配器不能仅根据 Sandbox Provider 的计算磁盘大小声明配额支持。容量上限不提供任何 I/O 公平性。 + +容量按对象准入。对于新绑定，Core 在绑定前根据部署请求验证所选配置的能力声明；随后若其他配置在此期间被选中，绑定事务以可重试的冲突拒绝。Core 将获准容量与引用一同持久化，并原样传给每次 `Create`。保留的绑定按其自身配置和获准容量验证，因此更改部署请求或选择不会调整现有存储的大小，也不会重新绑定。 + +设置[租户容量策略](configuration.md#tenant-capacity)后，新绑定还须按租户已承诺容量准入：即该租户所有尚未确认删除的工作区的获准容量之和，跨所有配置计算。创建中、失败或结果未确认以及删除中的工作区，在适配器确认删除前均计入。没有获准容量的保留工作区视为无上限，在其被删除前阻止该租户的进一步准入。在该策略下，未请求容量的请求不受支持；会超出策略的请求在分配任何存储或计算资源前以不可用失败。既有绑定绝不重新准入，调低策略也不会驱逐任何对象。 工作区仅允许单写入方访问。Core 必须串行管理计算资源归属，并在挂载替代计算资源前确认先前写入方已停止。本协议没有跨虚拟机锁能力：客户机内的 `flock` 不证明主机锁或对其他客户机的互斥。执行位置不得基于客户机本地锁宣称并发挂载安全。 @@ -51,13 +59,23 @@ source_hash: 21eb220ac59a477a8e799a6de20e01b339536901e8c9685695d2ac498b1b7bdb `nfs` 适配器使用由运维人员预挂载的 Linux 内核 NFS 文件系统。支持的配置是受信任 AUTH_SYS 客户端上的 NFSv4.2，启用 `root_squash`，Core 和节点使用相同导出和绝对挂载路径。适配器不挂载导出，也不管理 NFS 凭据或服务账户。其他操作系统返回 `ErrUnsupported`。[配置文档](./configuration.md#independent-workspace-storage)负责安装步骤和选择方式。 -参数精确包含 `root`、`namespace_id` 和 `uid`。`root` 是规范的绝对挂载路径；`namespace_id` 是非零规范 UUID；`uid` 是 Core 和节点均须使用的非零有效主机 UID。初始部署使用 65532。运维人员将挂载根目录设置为该 UID 所有、权限 `0700`，并准备同 UID 所有、权限 `0600` 的常规文件 `.oac-storage-root`，内容精确为命名空间 UUID 加一个换行符。应由导出服务端管理员在服务端存储上完成此操作；启用 root squash 的客户端不能假定有权变更所有者。根标记声明所选导出的身份，并非凭据或分布式锁。 +参数包含 `root`、`namespace_id`、`uid`，配额模式下还包含 `quota`。`root` 是规范的绝对挂载路径；`namespace_id` 是非零规范 UUID；`uid` 是 Core 和节点均须使用的非零有效主机 UID。初始部署使用 65532。运维人员将挂载根目录设置为该 UID 所有、权限 `0700`，并准备同 UID 所有、权限 `0600` 的常规文件 `.oac-storage-root`，内容精确为命名空间 UUID 加一个换行符。应由导出服务端管理员在服务端存储上完成此操作；启用 root squash 的客户端不能假定有权变更所有者。根标记声明所选导出的身份，并非凭据或分布式锁。 构造函数只验证并规范化 JSON，不执行文件系统 I/O。`Check` 验证打开的根目录确实位于内核 NFS，有效 UID 与私有所有权和权限匹配，命名空间标记匹配，并且实际文件描述符上的 `user.*` xattr 设置、读取和删除成功。包括打开根目录和可用性检查在内的所有文件系统操作，共享每进程最多 32 个在途操作的预算。调用者超时后收到 `ErrUnconfirmed`；该操作仍占用名额，直到内核调用返回。预算耗尽返回 `ErrUnavailable`。相对于目录描述符的操作始终锚定已打开的根目录。挂载不可用或配置路径只是本地目录时，验证失败，不创建本地回退。 -能力声明为 `host_directory`、`user_xattr: true`、`capacity_quota: false`。原生挂载凭据仅包含命名空间标识；解析时根据不可变配置和文件系统所有权验证完整 binding，随后返回根目录下的 `objects//live/data`。仅此数据目录暴露给 guest。租户和应用输入都不提供主机路径。 +能力声明为 `host_directory`、`user_xattr: true`，且仅在配额模式下 `capacity_quota` 为 true。原生挂载凭据仅包含命名空间标识；解析时根据不可变配置和文件系统所有权验证完整 binding，随后返回根目录下的 `objects//live/data`。仅此数据目录暴露给 guest。租户和应用输入都不提供主机路径。 + +未配置 `quota` 时，适配器不实施按 Environment 划分的字节或 inode 配额，也不需要凭据。单个对象可能耗尽共享文件系统，阻止其他对象写入，也可能阻止写入安全删除所需的持久终态元数据。运维人员必须监测可用字节和 inode，并为生命周期元数据和清理保留余量。 + +### 配额模式 {#quota-mode} + +配额模式通过 NFS 服务端自身文件系统强制执行的 XFS project quota 为每个对象设定上限。`quota` 精确包含 `endpoint`（存储服务器 SSH 服务的 `host:port`）、`user`（受限账户）和 `host_key`（以 authorized-key 格式固定的服务器 OpenSSH 公共主机密钥；注释会被丢弃）。凭据为 `{"ssh_private_key": "..."}`，即一个未加密的 OpenSSH 私钥，服务器仅为 `oac-workspace-quota` 强制命令授权其公钥。服务器准备工作由[配置文档](configuration.md#quota-mode-storage-server)负责。 + +对于正容量，`Create` 在认领标识并创建对象空的 `staging/` 和 `trash/` 之后、创建任何 stage 或返回现有 `live` 对象之前调用该命令，因此每次重试都会重新验证上限。请求携带完整引用、配置 ID、命名空间 ID、容量以及每 MiB 256 个的 inode 上限；请求无法指定任何路径、project ID、用户或命令。客户端验证固定的主机密钥，仅在内存中保存私钥，每次调用使用一个有界会话。请求发送前失败为 `ErrUnavailable`；发送后缺少或无法解析应答为 `ErrUnconfirmed`，同一引用在重试时收敛。`Check` 同样调用该命令，在选择前证明可达性、主机密钥、命名空间标记、日志库和强制执行状态。 + +该命令在存储服务器上以 root 运行。它持有一把锁，以不跟随链接的方式相对配置的导出打开 `objects//staging`，验证服务 UID、私有权限、设备及完整标识，拒绝已删除对象，并拒绝 `objects/` 或对象目录上的 project。同一 XFS 文件系统上、导出之外的本地 SQLite 日志库记录每个对象的引用、配置、容量、inode 上限、staging inode 及其 generation，以及从运维人员预留范围分配且永不复用的 project ID。分配先于任何变更提交；随后一次 `FS_IOC_FSSETXATTR` 在仍为空的 staging 目录上同时设置 project ID 和继承标志，服务器设置字节和 inode 硬限制（软限制为零）并回读验证。Stage 继承该 project；`live` 和退役的 trash 在重命名后保持该 project。标识、删除标记、对象目录和 `trash/` 父目录不属于该 project，因此已满的对象仍可标记删除并移除。移除会释放数据占用；保留的 `staging/` inode 仍计入该 project。属于其他文件系统或命名空间的日志库、未启用强制执行、范围耗尽、已有用户数据、外来或继承的 project、被替换的 staging inode、变更的引用、配置或容量，以及就绪对象上被修改或丢失的限制，均被拒绝而非修复。没有 dquot 的 project 读作零限制，绝不视为未使用的 ID。 -NFS 适配器不实施按 Environment 或租户划分的字节或 inode 配额。单个对象可能耗尽共享文件系统，阻止其他对象写入，也可能阻止写入安全删除所需的持久终态元数据。运维人员必须监测可用字节和 inode，并为生命周期元数据和清理保留余量。对整个命名空间设置容量上限，并不提供租户或对象之间的容量隔离。 +`Create` 成功时上限即具权威性；`Resolve` 不重新证明，因此恢复就绪工作区不依赖存储服务器的 SSH 服务。存储管理员事后禁用配额或清除限制会使强制执行失效，直至按支持的流程恢复。Project ID 和限制保存在 XFS 文件系统中，因此配额模式需要为其描述的[块级备份](getting-started/operations.md#back-up)。 每个对象拥有 `objects//identity`、`staging/`、`live/`、`trash/`，以及删除开始后的 `deleted-marker`。永久标识绑定租户、Environment 和对象 UUID。Create 先准备并同步唯一的私有 staging 封装目录，再原子发布其非空目录为 `live`；重放不会覆盖现有 live 数据。Delete 先持久记录对象本地的终态标记，将 live 数据退役到唯一的对象本地 trash，再先删除数据、后删除其所有权标记。并发删除者收敛。最少量的标识和删除元数据会有意保留；删除不代表所有元数据文件均消失。Create 和 Resolve 拒绝已退役标识。迟到的在途 Create 可能留下空的受控元数据，需要在其系统调用完成后重复 Delete；它不能授权新写入者或复用对象标识。清理只查看该对象的 staging 和 trash，不全量扫描命名空间，也不使用后台服务。 diff --git a/go.mod b/go.mod index e851cd189..6c13bc5ef 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( go.opentelemetry.io/otel/sdk v1.44.0 go.opentelemetry.io/otel/sdk/metric v1.44.0 go.opentelemetry.io/proto/otlp v1.10.0 + golang.org/x/crypto v0.56.0 golang.org/x/net v0.58.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 diff --git a/go.sum b/go.sum index 3ed07c81a..e0f04e381 100644 --- a/go.sum +++ b/go.sum @@ -127,6 +127,8 @@ go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpu go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= +golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= @@ -135,6 +137,8 @@ golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= diff --git a/packages/agents-client/src/generated/core-api.ts b/packages/agents-client/src/generated/core-api.ts index 41f6ffd14..fef191a79 100644 --- a/packages/agents-client/src/generated/core-api.ts +++ b/packages/agents-client/src/generated/core-api.ts @@ -116,12 +116,6 @@ export interface AdminauditPage { export const adminauditPageFields = ["data", "has_more", "next_cursor"] as const; export const attachmentKindValues = ["host_directory"] as const; export type AttachmentKind = (typeof attachmentKindValues)[number]; -export interface Configuration { - adapter: string; - id: string; - parameters: Record; -} -export const configurationFields = ["adapter", "id", "parameters"] as const; export interface ConfigurationDiscoveryInput { configuration?: Record; credential?: Record; @@ -830,6 +824,13 @@ export interface SandboxResources { } export const sandboxResourcesFields = ["cpus", "environment_disk_mib", "memory_mib", "root_disk_mib"] as const; export const sandboxResourcesRequired = ["cpus", "memory_mib"] as const; +export interface Selection { + adapter: string; + id: string; + parameters: Record; + tenant_capacity_mib: number; +} +export const selectionFields = ["adapter", "id", "parameters", "tenant_capacity_mib"] as const; export interface ServiceState { execution_owner: boolean | null; revision: string | null; @@ -884,6 +885,15 @@ export interface TurnDiagnostics { turn_id: string; } export const turnDiagnosticsFields = ["failure", "items", "items_truncated", "object", "session_id", "status", "turn_id"] as const; +export interface WorkspaceStorageInput { + adapter: string; + credential?: Record; + id: string; + parameters: Record; + tenant_capacity_mib: number; +} +export const workspaceStorageInputFields = ["adapter", "credential", "id", "parameters", "tenant_capacity_mib"] as const; +export const workspaceStorageInputRequired = ["adapter", "id", "parameters", "tenant_capacity_mib"] as const; export interface WriteauditAPIKey { id: string; kind: "issued"; diff --git a/scripts/build-core-distribution.sh b/scripts/build-core-distribution.sh index 37da251f5..a1e6fe70f 100755 --- a/scripts/build-core-distribution.sh +++ b/scripts/build-core-distribution.sh @@ -107,6 +107,8 @@ python3 scripts/core-distribution-manifest.py bootstraps "$bundle" "$source_epoc python3 scripts/core-distribution-manifest.py docs . "$bundle" "$revision" mkdir -p "$bundle/runtime" cp services/core/deploy/codex/seccomp.json "$bundle/runtime/" +mkdir -p "$bundle/storage" +install -m 0755 services/core/internal/workspacefs/nfs/quota/oac-workspace-quota "$bundle/storage/" cp LICENSE "$bundle/" OAC_DEV_BUILD_REVISION="$revision" scripts/build-core-image-context.sh "$stage/core" diff --git a/services/core/cmd/server/main.go b/services/core/cmd/server/main.go index c9be5a7ff..b08da178a 100644 --- a/services/core/cmd/server/main.go +++ b/services/core/cmd/server/main.go @@ -153,7 +153,7 @@ func run(config processconfig.Config) error { } sandboxProviders := providers.Builtin() workspaceProviders := workspaceproviders.New() - workspaceStore := workspacepg.New(units) + workspaceStore := workspacepg.New(units, credentialKey, config.InstallationID) // The placement rules are built once: the provider declarations and the // public URL never change while Core runs. placementRules, err := placement.NewRules(sandboxProviders, config.PublicOrigin.String()) @@ -245,7 +245,7 @@ func run(config processconfig.Config) error { } // From this call on the Worker closes the lease, even when it fails to start. worker, err = execution.StartWorker(ctx, dispatcher, execution.Owner{ - Workspaces: workspaces.NewExecution(workspaceStore, workspacepg.NewExecution(lease), workspaceProviders, lease), + Workspaces: workspaces.NewExecution(workspaceStore, workspacepg.NewExecution(lease, credentialKey, config.InstallationID), workspaceProviders, lease), Lease: lease, Deployment: deploymentExecution, Sessions: sessionExecution, diff --git a/services/core/internal/api/workspace_storage.go b/services/core/internal/api/workspace_storage.go index 7a07df5c8..03b86b5a0 100644 --- a/services/core/internal/api/workspace_storage.go +++ b/services/core/internal/api/workspace_storage.go @@ -2,6 +2,7 @@ package api import ( "context" + "encoding/json" "errors" "net/http" @@ -9,38 +10,49 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspaces" ) -// WorkspaceStorage reads and changes the independent filesystem configuration. -// Configure serializes selection changes and records the admin audit source. +// WorkspaceStorage reads and changes the independent filesystem selection and +// tenant capacity policy. Configure serializes selection changes and records +// the admin audit source. type WorkspaceStorage interface { - Configuration(context.Context) (workspacefs.Configuration, error) - Configure(context.Context, workspacefs.Configuration) (workspacefs.Configuration, error) + Selection(context.Context) (workspaces.Selection, error) + Configure(context.Context, workspaces.SelectionInput) (workspaces.Selection, error) +} + +// WorkspaceStorageInput selects an immutable configuration and the tenant +// capacity policy together. Credential is write-only and adapter-owned. +type WorkspaceStorageInput struct { + ID string `json:"id" format:"uuid" binding:"required"` + Adapter string `json:"adapter" binding:"required"` + Parameters json.RawMessage `json:"parameters" swaggertype:"object" binding:"required"` + Credential json.RawMessage `json:"credential,omitempty" swaggertype:"object"` + TenantCapacityMiB *uint32 `json:"tenant_capacity_mib" binding:"required"` } // @Summary Retrieve workspace storage configuration -// @Description Core key only. Returns the immutable selected filesystem configuration. No configuration returns 404. +// @Description Core key only. Returns the immutable selected filesystem configuration and the tenant capacity policy, never a credential. No configuration returns 404. // @Tags Workspace Storage // @Produce json // @Security DeploymentAdminAuth -// @Success 200 {object} workspacefs.Configuration +// @Success 200 {object} workspaces.Selection // @Failure 400,401,404,409,500,503 {object} CoreErrorResponse // @Router /core/v1/workspace-storage [get] func (h *Handler) getWorkspaceStorage(w http.ResponseWriter, r *http.Request) { - configuration, err := h.WorkspaceStorage.Configuration(r.Context()) + selection, err := h.WorkspaceStorage.Selection(r.Context()) if err != nil { writeWorkspaceStorageError(w, r, err) return } - writeJSON(w, http.StatusOK, configuration) + writeJSON(w, http.StatusOK, selection) } // @Summary Select workspace storage configuration -// @Description Core key only. Selects an immutable filesystem configuration. Reusing an ID requires the same adapter and parameters; conflicting ownership returns 409. +// @Description Core key only. Selects an immutable filesystem configuration and the tenant capacity policy in one change. Reusing an ID requires the same adapter, parameters and credential; an omitted credential retains the stored one. A positive tenant_capacity_mib requires, and is required by, a configuration that enforces capacity quotas. Conflicting ownership returns 409. // @Tags Workspace Storage // @Accept json // @Produce json // @Security DeploymentAdminAuth -// @Param body body workspacefs.Configuration true "Immutable workspace storage configuration" -// @Success 200 {object} workspacefs.Configuration +// @Param body body api.WorkspaceStorageInput true "Workspace storage selection" +// @Success 200 {object} workspaces.Selection // @Failure 400,401,404,409,500,503 {object} CoreErrorResponse // @Router /core/v1/workspace-storage [put] func (h *Handler) putWorkspaceStorage(w http.ResponseWriter, r *http.Request) { @@ -48,18 +60,26 @@ func (h *Handler) putWorkspaceStorage(w http.ResponseWriter, r *http.Request) { if !ok { return } - var configuration workspacefs.Configuration - if decodeInputObject(raw, &configuration, "id", "adapter", "parameters") != nil || configuration.Validate() != nil { + var input WorkspaceStorageInput + if decodeInputObject(raw, &input, "id", "adapter", "parameters", "credential", "tenant_capacity_mib") != nil || input.TenantCapacityMiB == nil { + writeWorkspaceStorageError(w, r, workspacefs.ErrInvalid) + return + } + selection := workspaces.SelectionInput{Selection: workspaces.Selection{ + Configuration: workspacefs.Configuration{ID: input.ID, Adapter: input.Adapter, Parameters: input.Parameters}, + TenantCapacityMiB: *input.TenantCapacityMiB, + }, Credential: input.Credential} + if selection.Configuration.Validate() != nil { writeWorkspaceStorageError(w, r, workspacefs.ErrInvalid) return } setAdminAuditSource(r, "") - configuration, err := h.WorkspaceStorage.Configure(r.Context(), configuration) + result, err := h.WorkspaceStorage.Configure(r.Context(), selection) if err != nil { writeWorkspaceStorageError(w, r, err) return } - writeJSON(w, http.StatusOK, configuration) + writeJSON(w, http.StatusOK, result) } func writeWorkspaceStorageError(w http.ResponseWriter, r *http.Request, err error) { diff --git a/services/core/internal/api/workspace_storage_test.go b/services/core/internal/api/workspace_storage_test.go index c15d9d783..43b524b80 100644 --- a/services/core/internal/api/workspace_storage_test.go +++ b/services/core/internal/api/workspace_storage_test.go @@ -15,25 +15,25 @@ import ( ) type fakeWorkspaceStorage struct { - t testing.TB - configuration func(context.Context) (workspacefs.Configuration, error) - configure func(context.Context, workspacefs.Configuration) (workspacefs.Configuration, error) + t testing.TB + selection func(context.Context) (workspaces.Selection, error) + configure func(context.Context, workspaces.SelectionInput) (workspaces.Selection, error) } -func (f *fakeWorkspaceStorage) Configuration(ctx context.Context) (workspacefs.Configuration, error) { - if f.configuration == nil { - f.t.Fatal("unexpected WorkspaceStorage.Configuration") +func (f *fakeWorkspaceStorage) Selection(ctx context.Context) (workspaces.Selection, error) { + if f.selection == nil { + f.t.Fatal("unexpected WorkspaceStorage.Selection") } - return f.configuration(ctx) + return f.selection(ctx) } -func (f *fakeWorkspaceStorage) Configure(ctx context.Context, c workspacefs.Configuration) (workspacefs.Configuration, error) { +func (f *fakeWorkspaceStorage) Configure(ctx context.Context, c workspaces.SelectionInput) (workspaces.Selection, error) { if f.configure == nil { f.t.Fatal("unexpected WorkspaceStorage.Configure") } return f.configure(ctx, c) } -const workspaceConfigurationJSON = `{"id":"11111111-1111-4111-8111-111111111111","adapter":"directory","parameters":{"root":"/workspace-storage"}}` +const workspaceConfigurationJSON = `{"id":"11111111-1111-4111-8111-111111111111","adapter":"directory","parameters":{"root":"/workspace-storage"},"tenant_capacity_mib":0}` func workspaceStorageRequest(h http.Handler, method, token, body string) *httptest.ResponseRecorder { request := httptest.NewRequest(method, "/core/v1/workspace-storage", strings.NewReader(body)) @@ -61,36 +61,44 @@ func TestWorkspaceStorageAuthentication(t *testing.T) { func TestWorkspaceStorageRoundTrip(t *testing.T) { deps, f := testDependencies(t) - var saved workspacefs.Configuration - f.workspaceStorage.configure = func(ctx context.Context, c workspacefs.Configuration) (workspacefs.Configuration, error) { + var saved workspaces.Selection + var credential json.RawMessage + f.workspaceStorage.configure = func(ctx context.Context, c workspaces.SelectionInput) (workspaces.Selection, error) { source, ok := adminaudit.FromContext(ctx) if !ok || source.CredentialID == "" || source.ProjectID != "" { t.Fatalf("missing deployment audit source: %+v", source) } - saved = c + saved, credential = c.Selection, c.Credential return saved, nil } - f.workspaceStorage.configuration = func(context.Context) (workspacefs.Configuration, error) { return saved, nil } + f.workspaceStorage.selection = func(context.Context) (workspaces.Selection, error) { return saved, nil } h := newTestHandler(t, deps) + body := strings.Replace(workspaceConfigurationJSON, `"tenant_capacity_mib":0`, `"tenant_capacity_mib":8192,"credential":{"ssh_private_key":"synthetic-private-key"}`, 1) for _, method := range []string{http.MethodPut, http.MethodGet} { - result := workspaceStorageRequest(h, method, "admin", workspaceConfigurationJSON) - if result.Code != http.StatusOK { + result := workspaceStorageRequest(h, method, "admin", body) + if result.Code != http.StatusOK || strings.Contains(result.Body.String(), "synthetic-private-key") || strings.Contains(result.Body.String(), "credential") { t.Fatalf("%s: %d %s", method, result.Code, result.Body.String()) } - var got workspacefs.Configuration + var got workspaces.Selection if err := json.Unmarshal(result.Body.Bytes(), &got); err != nil { t.Fatal(err) } - if got.ID != saved.ID || got.Adapter != saved.Adapter || string(got.Parameters) != string(saved.Parameters) { + if got.ID != saved.ID || got.Adapter != saved.Adapter || string(got.Parameters) != string(saved.Parameters) || got.TenantCapacityMiB != 8192 { t.Fatalf("configuration round trip: %+v", got) } } + if string(credential) != `{"ssh_private_key":"synthetic-private-key"}` { + t.Fatalf("write-only credential not forwarded: %s", credential) + } + if result := workspaceStorageRequest(h, http.MethodPut, "admin", workspaceConfigurationJSON); result.Code != http.StatusOK || credential != nil { + t.Fatalf("omitted credential must be absent, not empty: %d %q", result.Code, credential) + } } func TestWorkspaceStorageRejectsInvalidInput(t *testing.T) { deps, _ := testDependencies(t) h := newTestHandler(t, deps) - for _, body := range []string{"null", "{}", `{"id":"not-a-uuid","adapter":"directory","parameters":{}}`, strings.Replace(workspaceConfigurationJSON, `"parameters":{`, `"host_path":"/arbitrary","parameters":{`, 1), strings.Replace(workspaceConfigurationJSON, `"parameters":{"root":"/workspace-storage"}`, `"parameters":null`, 1), strings.Replace(workspaceConfigurationJSON, `"id":`, `"ID":`, 1)} { + for _, body := range []string{"null", "{}", `{"id":"not-a-uuid","adapter":"directory","parameters":{},"tenant_capacity_mib":0}`, strings.Replace(workspaceConfigurationJSON, `,"tenant_capacity_mib":0`, ``, 1), strings.Replace(workspaceConfigurationJSON, `"tenant_capacity_mib":0`, `"tenant_capacity_mib":-1`, 1), strings.Replace(workspaceConfigurationJSON, `"parameters":{`, `"host_path":"/arbitrary","parameters":{`, 1), strings.Replace(workspaceConfigurationJSON, `"parameters":{"root":"/workspace-storage"}`, `"parameters":null`, 1), strings.Replace(workspaceConfigurationJSON, `"id":`, `"ID":`, 1)} { result := workspaceStorageRequest(h, http.MethodPut, "admin", body) if result.Code != http.StatusBadRequest { t.Fatalf("%s: %d %s", body, result.Code, result.Body.String()) @@ -106,15 +114,15 @@ func TestWorkspaceStorageSafeErrors(t *testing.T) { {workspaces.ErrNotConfigured, 404}, {workspacefs.ErrNotFound, 404}, {workspacefs.ErrInvalid, 400}, {workspacefs.ErrUnsupported, 400}, {workspacefs.ErrOwnership, 409}, {workspaces.ErrConflict, 409}, - {workspacefs.ErrUnavailable, 503}, {workspacefs.ErrUnconfirmed, 503}, + {workspacefs.ErrUnavailable, 503}, {workspacefs.ErrUnconfirmed, 503}, {workspaces.ErrCapacityExhausted, 503}, {fmt.Errorf("unknown /secret/native/path"), 500}, } { t.Run(tc.err.Error(), func(t *testing.T) { deps, f := testDependencies(t) err := fmt.Errorf("/secret/native/path: %w", tc.err) - f.workspaceStorage.configuration = func(context.Context) (workspacefs.Configuration, error) { return workspacefs.Configuration{}, err } - f.workspaceStorage.configure = func(context.Context, workspacefs.Configuration) (workspacefs.Configuration, error) { - return workspacefs.Configuration{}, err + f.workspaceStorage.selection = func(context.Context) (workspaces.Selection, error) { return workspaces.Selection{}, err } + f.workspaceStorage.configure = func(context.Context, workspaces.SelectionInput) (workspaces.Selection, error) { + return workspaces.Selection{}, err } h := newTestHandler(t, deps) for _, method := range []string{http.MethodGet, http.MethodPut} { diff --git a/services/core/internal/credentialcrypto/workspace_configuration.go b/services/core/internal/credentialcrypto/workspace_configuration.go new file mode 100644 index 000000000..e1546bbca --- /dev/null +++ b/services/core/internal/credentialcrypto/workspace_configuration.go @@ -0,0 +1,32 @@ +package credentialcrypto + +import ( + "encoding/json" + "unicode/utf8" +) + +// A workspace filesystem credential is bound to its installation and to the +// immutable configuration that owns it, so it cannot be read as another +// configuration's or another installation's credential. +func workspaceConfigurationData(installation, configuration string) ([]byte, error) { + if installation == "" || configuration == "" || !utf8.ValidString(installation) || !utf8.ValidString(configuration) { + return nil, errInvalidBinding + } + return json.Marshal([]string{"oac.workspace-configuration.v1", installation, configuration}) +} + +func (c *Cipher) SealWorkspaceConfiguration(plaintext []byte, installation, configuration string) ([]byte, error) { + aad, err := workspaceConfigurationData(installation, configuration) + if err != nil { + return nil, err + } + return c.seal(plaintext, aad) +} + +func (c *Cipher) OpenWorkspaceConfiguration(ciphertext []byte, installation, configuration string) ([]byte, error) { + aad, err := workspaceConfigurationData(installation, configuration) + if err != nil { + return nil, err + } + return c.open(ciphertext, aad) +} diff --git a/services/core/internal/credentialcrypto/workspace_configuration_test.go b/services/core/internal/credentialcrypto/workspace_configuration_test.go new file mode 100644 index 000000000..2ee68f494 --- /dev/null +++ b/services/core/internal/credentialcrypto/workspace_configuration_test.go @@ -0,0 +1,32 @@ +package credentialcrypto + +import ( + "bytes" + "testing" +) + +func TestWorkspaceConfigurationCredentialBinding(t *testing.T) { + c, err := New(bytes.Repeat([]byte{9}, 32)) + if err != nil { + t.Fatal(err) + } + encrypted, err := c.SealWorkspaceConfiguration([]byte("private-key"), "installation", "configuration") + if err != nil { + t.Fatal(err) + } + for _, binding := range [][2]string{{"other", "configuration"}, {"installation", "other"}, {"configuration", "installation"}} { + if _, err := c.OpenWorkspaceConfiguration(encrypted, binding[0], binding[1]); err == nil { + t.Fatal("accepted different workspace configuration binding") + } + } + if _, err := c.OpenSandboxDeployment(encrypted, "installation", 1); err == nil { + t.Fatal("accepted a workspace credential as a sandbox credential") + } + if _, err := c.SealWorkspaceConfiguration([]byte("x"), "", "configuration"); err == nil { + t.Fatal("sealed without an installation") + } + plain, err := c.OpenWorkspaceConfiguration(encrypted, "installation", "configuration") + if err != nil || string(plain) != "private-key" { + t.Fatal("credential did not roundtrip", err) + } +} diff --git a/services/core/internal/db/queries/workspaces.sql b/services/core/internal/db/queries/workspaces.sql index 1d547f81a..7a285939e 100644 --- a/services/core/internal/db/queries/workspaces.sql +++ b/services/core/internal/db/queries/workspaces.sql @@ -2,13 +2,37 @@ LOCK TABLE workspace_fs_configurations IN SHARE ROW EXCLUSIVE MODE; -- name: InsertWorkspaceConfiguration :exec -INSERT INTO workspace_fs_configurations (id, adapter, parameters) VALUES ($1, $2, $3) +INSERT INTO workspace_fs_configurations (id, adapter, parameters, credential) VALUES ($1, $2, $3, $4) ON CONFLICT (id) DO NOTHING; -- name: MatchWorkspaceConfiguration :one -SELECT (adapter = sqlc.arg(adapter) AND parameters = sqlc.arg(parameters)::jsonb)::boolean AS matches +SELECT (adapter = sqlc.arg(adapter) AND parameters = sqlc.arg(parameters)::jsonb)::boolean AS matches, credential FROM workspace_fs_configurations WHERE id = sqlc.arg(id); +-- name: GetWorkspaceConfiguration :one +SELECT * FROM workspace_fs_configurations WHERE id = $1; + +-- name: SetWorkspaceTenantCapacity :exec +UPDATE workspace_storage_admission SET tenant_capacity_mib = $1; + +-- name: GetWorkspaceSelection :one +SELECT sqlc.embed(c), a.tenant_capacity_mib +FROM workspace_storage_admission a JOIN workspace_fs_configurations c ON c.active; + +-- name: LockWorkspaceAdmission :one +SELECT tenant_capacity_mib FROM workspace_storage_admission FOR SHARE; + +-- name: LockWorkspaceTenant :exec +SELECT pg_advisory_xact_lock(hashtext('oac.workspace-tenant-capacity'), hashtext(sqlc.arg(tenant_id)::uuid::text)); + +-- name: TenantWorkspaceCapacity :one +SELECT COALESCE(SUM(w.capacity_mib), 0)::bigint AS committed_mib, + COALESCE(bool_or(w.capacity_mib = 0), false)::boolean AS unbounded +FROM environment_workspaces w +JOIN environments e ON e.id = w.environment_id +JOIN sessions s ON s.id = e.session_id +WHERE s.tenant_id = $1 AND w.state <> 'deleted'; + -- name: ClearActiveWorkspaceConfiguration :exec UPDATE workspace_fs_configurations SET active = false WHERE active; @@ -34,8 +58,8 @@ JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = $1 AND e.id = $2; -- name: InsertEnvironmentWorkspace :exec -INSERT INTO environment_workspaces (object_id, environment_id, configuration_id) -VALUES ($1, $2, $3); +INSERT INTO environment_workspaces (object_id, environment_id, configuration_id, capacity_mib) +VALUES ($1, $2, $3, $4); -- name: MarkWorkspaceReady :execrows UPDATE environment_workspaces SET state = 'ready', attachment = $2 diff --git a/services/core/internal/db/sqlc/models.go b/services/core/internal/db/sqlc/models.go index d011c8983..fd8071a64 100644 --- a/services/core/internal/db/sqlc/models.go +++ b/services/core/internal/db/sqlc/models.go @@ -151,6 +151,7 @@ type EnvironmentWorkspace struct { ConfigurationID pgtype.UUID `json:"configuration_id"` State string `json:"state"` Attachment []byte `json:"attachment"` + CapacityMib int64 `json:"capacity_mib"` } type ExecutionProjectScope struct { @@ -564,6 +565,12 @@ type WorkspaceFsConfiguration struct { Adapter string `json:"adapter"` Parameters []byte `json:"parameters"` Active bool `json:"active"` + Credential []byte `json:"credential"` +} + +type WorkspaceStorageAdmission struct { + Singleton bool `json:"singleton"` + TenantCapacityMib int64 `json:"tenant_capacity_mib"` } type WriteAuditOperation struct { diff --git a/services/core/internal/db/sqlc/workspaces.sql.go b/services/core/internal/db/sqlc/workspaces.sql.go index e7549529e..90048ef68 100644 --- a/services/core/internal/db/sqlc/workspaces.sql.go +++ b/services/core/internal/db/sqlc/workspaces.sql.go @@ -43,7 +43,7 @@ func (q *Queries) ClearActiveWorkspaceConfiguration(ctx context.Context) error { } const getActiveWorkspaceConfiguration = `-- name: GetActiveWorkspaceConfiguration :one -SELECT id, adapter, parameters, active FROM workspace_fs_configurations WHERE active +SELECT id, adapter, parameters, active, credential FROM workspace_fs_configurations WHERE active ` func (q *Queries) GetActiveWorkspaceConfiguration(ctx context.Context) (WorkspaceFsConfiguration, error) { @@ -54,12 +54,13 @@ func (q *Queries) GetActiveWorkspaceConfiguration(ctx context.Context) (Workspac &i.Adapter, &i.Parameters, &i.Active, + &i.Credential, ) return i, err } const getEnvironmentWorkspace = `-- name: GetEnvironmentWorkspace :one -SELECT w.object_id, w.environment_id, w.configuration_id, w.state, w.attachment, c.id, c.adapter, c.parameters, c.active, s.tenant_id +SELECT w.object_id, w.environment_id, w.configuration_id, w.state, w.attachment, w.capacity_mib, c.id, c.adapter, c.parameters, c.active, c.credential, s.tenant_id FROM environment_workspaces w JOIN workspace_fs_configurations c ON c.id = w.configuration_id JOIN environments e ON e.id = w.environment_id @@ -87,33 +88,82 @@ func (q *Queries) GetEnvironmentWorkspace(ctx context.Context, arg GetEnvironmen &i.EnvironmentWorkspace.ConfigurationID, &i.EnvironmentWorkspace.State, &i.EnvironmentWorkspace.Attachment, + &i.EnvironmentWorkspace.CapacityMib, &i.WorkspaceFsConfiguration.ID, &i.WorkspaceFsConfiguration.Adapter, &i.WorkspaceFsConfiguration.Parameters, &i.WorkspaceFsConfiguration.Active, + &i.WorkspaceFsConfiguration.Credential, &i.TenantID, ) return i, err } +const getWorkspaceConfiguration = `-- name: GetWorkspaceConfiguration :one +SELECT id, adapter, parameters, active, credential FROM workspace_fs_configurations WHERE id = $1 +` + +func (q *Queries) GetWorkspaceConfiguration(ctx context.Context, id pgtype.UUID) (WorkspaceFsConfiguration, error) { + row := q.db.QueryRow(ctx, getWorkspaceConfiguration, id) + var i WorkspaceFsConfiguration + err := row.Scan( + &i.ID, + &i.Adapter, + &i.Parameters, + &i.Active, + &i.Credential, + ) + return i, err +} + +const getWorkspaceSelection = `-- name: GetWorkspaceSelection :one +SELECT c.id, c.adapter, c.parameters, c.active, c.credential, a.tenant_capacity_mib +FROM workspace_storage_admission a JOIN workspace_fs_configurations c ON c.active +` + +type GetWorkspaceSelectionRow struct { + WorkspaceFsConfiguration WorkspaceFsConfiguration `json:"workspace_fs_configuration"` + TenantCapacityMib int64 `json:"tenant_capacity_mib"` +} + +func (q *Queries) GetWorkspaceSelection(ctx context.Context) (GetWorkspaceSelectionRow, error) { + row := q.db.QueryRow(ctx, getWorkspaceSelection) + var i GetWorkspaceSelectionRow + err := row.Scan( + &i.WorkspaceFsConfiguration.ID, + &i.WorkspaceFsConfiguration.Adapter, + &i.WorkspaceFsConfiguration.Parameters, + &i.WorkspaceFsConfiguration.Active, + &i.WorkspaceFsConfiguration.Credential, + &i.TenantCapacityMib, + ) + return i, err +} + const insertEnvironmentWorkspace = `-- name: InsertEnvironmentWorkspace :exec -INSERT INTO environment_workspaces (object_id, environment_id, configuration_id) -VALUES ($1, $2, $3) +INSERT INTO environment_workspaces (object_id, environment_id, configuration_id, capacity_mib) +VALUES ($1, $2, $3, $4) ` type InsertEnvironmentWorkspaceParams struct { ObjectID pgtype.UUID `json:"object_id"` EnvironmentID pgtype.UUID `json:"environment_id"` ConfigurationID pgtype.UUID `json:"configuration_id"` + CapacityMib int64 `json:"capacity_mib"` } func (q *Queries) InsertEnvironmentWorkspace(ctx context.Context, arg InsertEnvironmentWorkspaceParams) error { - _, err := q.db.Exec(ctx, insertEnvironmentWorkspace, arg.ObjectID, arg.EnvironmentID, arg.ConfigurationID) + _, err := q.db.Exec(ctx, insertEnvironmentWorkspace, + arg.ObjectID, + arg.EnvironmentID, + arg.ConfigurationID, + arg.CapacityMib, + ) return err } const insertWorkspaceConfiguration = `-- name: InsertWorkspaceConfiguration :exec -INSERT INTO workspace_fs_configurations (id, adapter, parameters) VALUES ($1, $2, $3) +INSERT INTO workspace_fs_configurations (id, adapter, parameters, credential) VALUES ($1, $2, $3, $4) ON CONFLICT (id) DO NOTHING ` @@ -121,15 +171,21 @@ type InsertWorkspaceConfigurationParams struct { ID pgtype.UUID `json:"id"` Adapter string `json:"adapter"` Parameters []byte `json:"parameters"` + Credential []byte `json:"credential"` } func (q *Queries) InsertWorkspaceConfiguration(ctx context.Context, arg InsertWorkspaceConfigurationParams) error { - _, err := q.db.Exec(ctx, insertWorkspaceConfiguration, arg.ID, arg.Adapter, arg.Parameters) + _, err := q.db.Exec(ctx, insertWorkspaceConfiguration, + arg.ID, + arg.Adapter, + arg.Parameters, + arg.Credential, + ) return err } const listWorkspaceDeletionCandidates = `-- name: ListWorkspaceDeletionCandidates :many -SELECT w.object_id, w.environment_id, w.configuration_id, w.state, w.attachment, c.id, c.adapter, c.parameters, c.active, s.tenant_id +SELECT w.object_id, w.environment_id, w.configuration_id, w.state, w.attachment, w.capacity_mib, c.id, c.adapter, c.parameters, c.active, c.credential, s.tenant_id FROM environment_workspaces w JOIN workspace_fs_configurations c ON c.id = w.configuration_id JOIN environments e ON e.id = w.environment_id @@ -163,10 +219,12 @@ func (q *Queries) ListWorkspaceDeletionCandidates(ctx context.Context, objectID &i.EnvironmentWorkspace.ConfigurationID, &i.EnvironmentWorkspace.State, &i.EnvironmentWorkspace.Attachment, + &i.EnvironmentWorkspace.CapacityMib, &i.WorkspaceFsConfiguration.ID, &i.WorkspaceFsConfiguration.Adapter, &i.WorkspaceFsConfiguration.Parameters, &i.WorkspaceFsConfiguration.Active, + &i.WorkspaceFsConfiguration.Credential, &i.TenantID, ); err != nil { return nil, err @@ -179,6 +237,17 @@ func (q *Queries) ListWorkspaceDeletionCandidates(ctx context.Context, objectID return items, nil } +const lockWorkspaceAdmission = `-- name: LockWorkspaceAdmission :one +SELECT tenant_capacity_mib FROM workspace_storage_admission FOR SHARE +` + +func (q *Queries) LockWorkspaceAdmission(ctx context.Context) (int64, error) { + row := q.db.QueryRow(ctx, lockWorkspaceAdmission) + var tenant_capacity_mib int64 + err := row.Scan(&tenant_capacity_mib) + return tenant_capacity_mib, err +} + const lockWorkspaceConfigurations = `-- name: LockWorkspaceConfigurations :exec LOCK TABLE workspace_fs_configurations IN SHARE ROW EXCLUSIVE MODE ` @@ -205,6 +274,15 @@ func (q *Queries) LockWorkspaceSession(ctx context.Context, arg LockWorkspaceSes return id, err } +const lockWorkspaceTenant = `-- name: LockWorkspaceTenant :exec +SELECT pg_advisory_xact_lock(hashtext('oac.workspace-tenant-capacity'), hashtext($1::uuid::text)) +` + +func (q *Queries) LockWorkspaceTenant(ctx context.Context, tenantID pgtype.UUID) error { + _, err := q.db.Exec(ctx, lockWorkspaceTenant, tenantID) + return err +} + const markWorkspaceDeleted = `-- name: MarkWorkspaceDeleted :execrows UPDATE environment_workspaces SET state = 'deleted' WHERE object_id = $1 AND state = 'deleting' @@ -237,7 +315,7 @@ func (q *Queries) MarkWorkspaceReady(ctx context.Context, arg MarkWorkspaceReady } const matchWorkspaceConfiguration = `-- name: MatchWorkspaceConfiguration :one -SELECT (adapter = $1 AND parameters = $2::jsonb)::boolean AS matches +SELECT (adapter = $1 AND parameters = $2::jsonb)::boolean AS matches, credential FROM workspace_fs_configurations WHERE id = $3 ` @@ -247,11 +325,46 @@ type MatchWorkspaceConfigurationParams struct { ID pgtype.UUID `json:"id"` } -func (q *Queries) MatchWorkspaceConfiguration(ctx context.Context, arg MatchWorkspaceConfigurationParams) (bool, error) { +type MatchWorkspaceConfigurationRow struct { + Matches bool `json:"matches"` + Credential []byte `json:"credential"` +} + +func (q *Queries) MatchWorkspaceConfiguration(ctx context.Context, arg MatchWorkspaceConfigurationParams) (MatchWorkspaceConfigurationRow, error) { row := q.db.QueryRow(ctx, matchWorkspaceConfiguration, arg.Adapter, arg.Parameters, arg.ID) - var matches bool - err := row.Scan(&matches) - return matches, err + var i MatchWorkspaceConfigurationRow + err := row.Scan(&i.Matches, &i.Credential) + return i, err +} + +const setWorkspaceTenantCapacity = `-- name: SetWorkspaceTenantCapacity :exec +UPDATE workspace_storage_admission SET tenant_capacity_mib = $1 +` + +func (q *Queries) SetWorkspaceTenantCapacity(ctx context.Context, tenantCapacityMib int64) error { + _, err := q.db.Exec(ctx, setWorkspaceTenantCapacity, tenantCapacityMib) + return err +} + +const tenantWorkspaceCapacity = `-- name: TenantWorkspaceCapacity :one +SELECT COALESCE(SUM(w.capacity_mib), 0)::bigint AS committed_mib, + COALESCE(bool_or(w.capacity_mib = 0), false)::boolean AS unbounded +FROM environment_workspaces w +JOIN environments e ON e.id = w.environment_id +JOIN sessions s ON s.id = e.session_id +WHERE s.tenant_id = $1 AND w.state <> 'deleted' +` + +type TenantWorkspaceCapacityRow struct { + CommittedMib int64 `json:"committed_mib"` + Unbounded bool `json:"unbounded"` +} + +func (q *Queries) TenantWorkspaceCapacity(ctx context.Context, tenantID pgtype.UUID) (TenantWorkspaceCapacityRow, error) { + row := q.db.QueryRow(ctx, tenantWorkspaceCapacity, tenantID) + var i TenantWorkspaceCapacityRow + err := row.Scan(&i.CommittedMib, &i.Unbounded) + return i, err } const workspaceComputeReleased = `-- name: WorkspaceComputeReleased :one diff --git a/services/core/internal/deployment/service.go b/services/core/internal/deployment/service.go index 022a0d359..4bc68a72b 100644 --- a/services/core/internal/deployment/service.go +++ b/services/core/internal/deployment/service.go @@ -305,12 +305,11 @@ func (s *Service) ValidateWorkspaceConfiguration(ctx context.Context, declaratio if adapter.Policy.Workspace == nil { return workspacefs.ErrUnsupported } - capacity := uint32(0) - if setup.Specification.Workspace != nil { - if *setup.Specification.Workspace != declaration { - return workspacefs.ErrUnsupported - } - capacity = setup.Specification.Resources.EnvironmentDiskMiB + // An external deployment may change only quota semantics: workspaces keep + // their own admitted capacity, and new ones are validated against the + // selected declaration until the deployment's capacity request matches it. + if current := setup.Specification.Workspace; current != nil && (current.Attachment != declaration.Attachment || current.UserXAttr != declaration.UserXAttr) { + return workspacefs.ErrUnsupported } - return workspacefs.ValidateCombination(*adapter.Policy.Workspace, declaration, capacity) + return workspacefs.ValidateCombination(*adapter.Policy.Workspace, declaration, 0) } diff --git a/services/core/internal/deployment/workspace_configuration_test.go b/services/core/internal/deployment/workspace_configuration_test.go index 594687261..32c427544 100644 --- a/services/core/internal/deployment/workspace_configuration_test.go +++ b/services/core/internal/deployment/workspace_configuration_test.go @@ -43,8 +43,13 @@ func TestWorkspaceConfigurationRejectsChangedExternalReceiptAndReadFailure(t *te if err := service.ValidateWorkspaceConfiguration(t.Context(), declaration); err != nil { t.Fatal(err) } + quota := declaration + quota.CapacityQuota = true + if err := service.ValidateWorkspaceConfiguration(t.Context(), quota); err != nil { + t.Fatal("quota semantics change refused", err) + } changed := declaration - changed.CapacityQuota = true + changed.UserXAttr = false if err := service.ValidateWorkspaceConfiguration(t.Context(), changed); !errors.Is(err, workspacefs.ErrUnsupported) { t.Fatal("changed receipt accepted", err) } diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 71567378d..2edb40384 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -273,7 +273,7 @@ func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.Sandbox } var workspace *workspacefs.Binding if spec.Workspace != nil { - workspace, err = r.workspaces.GetReady(ctx, owner.TenantID, owner.EnvironmentID, r.config.WorkspaceRequirements, spec.Resources.EnvironmentDiskMiB) + workspace, err = r.workspaces.GetReady(ctx, owner.TenantID, owner.EnvironmentID, r.config.WorkspaceRequirements) if err == nil && workspace == nil { err = workspacefs.ErrNotFound } diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index a4a14cc11..278d30776 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -215,7 +215,7 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p return deployment.Allocation{}, err } } else if existing.State == "released" { - workspace, err = r.workspaces.GetReady(ctx, tenant, environment, r.config.WorkspaceRequirements, spec.Resources.EnvironmentDiskMiB) + workspace, err = r.workspaces.GetReady(ctx, tenant, environment, r.config.WorkspaceRequirements) if err != nil { return deployment.Allocation{}, err } diff --git a/services/core/internal/execution/runtime_workspace.go b/services/core/internal/execution/runtime_workspace.go index 3f4e617e2..7f514164f 100644 --- a/services/core/internal/execution/runtime_workspace.go +++ b/services/core/internal/execution/runtime_workspace.go @@ -8,26 +8,26 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspaces" ) -func (w *Worker) Configuration(ctx context.Context) (workspacefs.Configuration, error) { +func (w *Worker) Selection(ctx context.Context) (workspaces.Selection, error) { if w.runtimes.workspaces == nil { - return workspacefs.Configuration{}, workspaces.ErrNotConfigured + return workspaces.Selection{}, workspaces.ErrNotConfigured } - return w.runtimes.workspaces.Configuration(ctx) + return w.runtimes.workspaces.Selection(ctx) } -func (w *Worker) Configure(ctx context.Context, configuration workspacefs.Configuration) (workspacefs.Configuration, error) { +func (w *Worker) Configure(ctx context.Context, input workspaces.SelectionInput) (workspaces.Selection, error) { m := w.runtimes if m.workspaces == nil { - return workspacefs.Configuration{}, workspaces.ErrNotConfigured + return workspaces.Selection{}, workspaces.ErrNotConfigured } ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() unlock, err := m.lockMutation(ctx) if err != nil { - return workspacefs.Configuration{}, err + return workspaces.Selection{}, err } defer unlock() - return m.workspaces.Configure(ctx, configuration, func(declaration workspacefs.Declaration) error { + return m.workspaces.Configure(ctx, input, func(declaration workspacefs.Declaration) error { return m.deploymentService.ValidateWorkspaceConfiguration(ctx, declaration) }) } diff --git a/services/core/internal/execution/runtime_workspace_settlement_test.go b/services/core/internal/execution/runtime_workspace_settlement_test.go index 6207c92df..e6908caee 100644 --- a/services/core/internal/execution/runtime_workspace_settlement_test.go +++ b/services/core/internal/execution/runtime_workspace_settlement_test.go @@ -45,7 +45,7 @@ type settlementWorkspaceControl struct { func (c *settlementWorkspaceControl) Declaration() workspacefs.Declaration { return workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true} } -func (c *settlementWorkspaceControl) Create(_ context.Context, ref workspacefs.Reference) (workspacefs.Attachment, error) { +func (c *settlementWorkspaceControl) Create(_ context.Context, ref workspacefs.Reference, _ uint32) (workspacefs.Attachment, error) { return workspacefs.Attachment{Reference: ref, ConfigurationID: c.configuration.ID, Kind: workspacefs.AttachmentHostDirectory, Native: []byte(`{}`)}, nil } func (c *settlementWorkspaceControl) Delete(context.Context, workspacefs.Reference) error { @@ -55,11 +55,11 @@ func (c *settlementWorkspaceControl) Delete(context.Context, workspacefs.Referen type settlementWorkspaceControls struct{ control *settlementWorkspaceControl } -func (c settlementWorkspaceControls) Control(workspacefs.Configuration) (workspacefs.Control, error) { +func (c settlementWorkspaceControls) Control(workspacefs.ConfigurationRecord) (workspacefs.Control, error) { return c.control, nil } -func (c settlementWorkspaceControls) Normalize(config workspacefs.Configuration) (workspacefs.Configuration, error) { - return config, nil +func (c settlementWorkspaceControls) Normalize(config workspacefs.Configuration, _ json.RawMessage) (workspacefs.ConfigurationRecord, error) { + return workspacefs.ConfigurationRecord{Configuration: config}, nil } type workspaceSettlementFixture struct { @@ -114,10 +114,10 @@ func workspaceSettlementFixtureWithSetup(t *testing.T, provider sandbox.SandboxP } } configuration := workspacefs.Configuration{ID: uuid.NewString(), Adapter: "fixture", Parameters: []byte(`{}`)} - storage := workspacepg.New(pgunit.NewPool(pool)) - writer := workspacepg.NewExecution(lease) + storage := workspacepg.New(pgunit.NewPool(pool), pgtest.CredentialKey(t), installation) + writer := workspacepg.NewExecution(lease, pgtest.CredentialKey(t), installation) workspaceAudit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", RequestID: uuid.NewString(), TraceID: uuid.NewString()}) - if err = writer.SelectConfiguration(workspaceAudit, configuration); err != nil { + if err = writer.SelectConfiguration(workspaceAudit, workspacefs.ConfigurationRecord{Configuration: configuration}, 0); err != nil { t.Fatal(err) } control := &settlementWorkspaceControl{configuration: configuration} diff --git a/services/core/internal/execution/runtime_workspace_test.go b/services/core/internal/execution/runtime_workspace_test.go index cd5d1cc1a..2b60c1af2 100644 --- a/services/core/internal/execution/runtime_workspace_test.go +++ b/services/core/internal/execution/runtime_workspace_test.go @@ -2,6 +2,7 @@ package execution import ( "context" + "encoding/json" "errors" "testing" @@ -29,17 +30,23 @@ type executionWorkspaceFixture struct { failure error } -func (f *executionWorkspaceFixture) Bind(context.Context, string, string) (workspaces.Record, error) { +func (f *executionWorkspaceFixture) Get(context.Context, string, string) (workspaces.Record, error) { + return f.record, nil +} +func (f *executionWorkspaceFixture) Configuration(context.Context, string) (workspacefs.ConfigurationRecord, error) { + return workspacefs.ConfigurationRecord{}, workspaces.ErrNotFound +} +func (f *executionWorkspaceFixture) Bind(context.Context, string, string, string, uint32) (workspaces.Record, error) { f.binds++ return f.record, nil } -func (f *executionWorkspaceFixture) ControlForConfiguration(workspacefs.Configuration) (workspacefs.Control, error) { +func (f *executionWorkspaceFixture) ControlForConfiguration(workspacefs.ConfigurationRecord) (workspacefs.Control, error) { return f, nil } func (f *executionWorkspaceFixture) Declaration() workspacefs.Declaration { return workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory} } -func (f *executionWorkspaceFixture) Create(context.Context, workspacefs.Reference) (workspacefs.Attachment, error) { +func (f *executionWorkspaceFixture) Create(context.Context, workspacefs.Reference, uint32) (workspacefs.Attachment, error) { f.creates++ return workspacefs.Attachment{Reference: f.record.Reference, ConfigurationID: f.record.Configuration.ID, Kind: workspacefs.AttachmentHostDirectory, Native: []byte(`{}`)}, f.failure } @@ -66,15 +73,15 @@ func (f *executionWorkspaceFixture) MarkDeleted(context.Context, workspacefs.Ref type workspaceControls struct{ fixture *executionWorkspaceFixture } -func (c workspaceControls) Normalize(configuration workspacefs.Configuration) (workspacefs.Configuration, error) { - return configuration, nil +func (c workspaceControls) Normalize(configuration workspacefs.Configuration, _ json.RawMessage) (workspacefs.ConfigurationRecord, error) { + return workspacefs.ConfigurationRecord{Configuration: configuration}, nil } -func (c workspaceControls) Control(config workspacefs.Configuration) (workspacefs.Control, error) { +func (c workspaceControls) Control(config workspacefs.ConfigurationRecord) (workspacefs.Control, error) { return c.fixture.ControlForConfiguration(config) } func newExecutionWorkspaceFixture() *executionWorkspaceFixture { - return &executionWorkspaceFixture{record: workspaces.Record{Reference: workspacefs.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), ObjectID: uuid.NewString()}, Configuration: workspacefs.Configuration{ID: uuid.NewString(), Adapter: "fixture", Parameters: []byte(`{}`)}, State: workspaces.Creating}} + return &executionWorkspaceFixture{record: workspaces.Record{Reference: workspacefs.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), ObjectID: uuid.NewString()}, Configuration: workspacefs.ConfigurationRecord{Configuration: workspacefs.Configuration{ID: uuid.NewString(), Adapter: "fixture", Parameters: []byte(`{}`)}}, State: workspaces.Creating}} } func TestRuntimeWorkspaceConvergesBeforeComputeReservation(t *testing.T) { @@ -125,10 +132,10 @@ func TestRuntimeWorkspaceScanRunsWithoutSandboxOrNodes(t *testing.T) { } } -func (f *executionWorkspaceFixture) ActiveConfiguration(context.Context) (workspacefs.Configuration, error) { +func (f *executionWorkspaceFixture) ActiveConfiguration(context.Context) (workspacefs.ConfigurationRecord, error) { return f.record.Configuration, nil } -func (f *executionWorkspaceFixture) SelectConfiguration(_ context.Context, c workspacefs.Configuration) error { +func (f *executionWorkspaceFixture) SelectConfiguration(_ context.Context, c workspacefs.ConfigurationRecord, _ uint32) error { f.record.Configuration = c return nil } @@ -174,9 +181,9 @@ func TestConfigureWorkspaceBeforeSandboxSetup(t *testing.T) { } defer m.stop() worker := &Worker{runtimes: m} - next := f.record.Configuration + next := f.record.Configuration.Configuration next.ID = uuid.NewString() - got, err := worker.Configure(t.Context(), next) + got, err := worker.Configure(t.Context(), workspaces.SelectionInput{Selection: workspaces.Selection{Configuration: next}}) if err != nil || got.ID != next.ID { t.Fatal("filesystem selection requires sandbox setup", err) } diff --git a/services/core/internal/persistence/postgres/workspacepg/capacity_test.go b/services/core/internal/persistence/postgres/workspacepg/capacity_test.go new file mode 100644 index 000000000..49c49f8e8 --- /dev/null +++ b/services/core/internal/persistence/postgres/workspacepg/capacity_test.go @@ -0,0 +1,200 @@ +package workspacepg_test + +import ( + "bytes" + "errors" + "strings" + "sync" + "testing" + + "github.com/google/uuid" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/workspacepg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspaces" +) + +const syntheticKey = "synthetic-private-ssh-key" + +func credentialed() workspacefs.ConfigurationRecord { + c := newConfiguration() + c.Credential = []byte(`{"ssh_private_key":"` + syntheticKey + `"}`) + return c +} + +func TestConfigurationCredentialIsSealedAndWriteOnly(t *testing.T) { + f := newFixture(t) + config := credentialed() + f.selectConfiguration(t, config, 4096) + var stored []byte + var audit string + if err := f.pool.QueryRow(t.Context(), `SELECT credential, (SELECT string_agg(row_to_json(a)::text, '') FROM admin_audit_log a) FROM workspace_fs_configurations WHERE id=$1`, config.ID).Scan(&stored, &audit); err != nil { + t.Fatal(err) + } + if len(stored) == 0 || bytes.Contains(stored, []byte(syntheticKey)) || strings.Contains(audit, syntheticKey) { + t.Fatal("credential stored or audited in plaintext") + } + active, err := f.store.ActiveConfiguration(t.Context()) + if err != nil || !bytes.Equal(active.Credential, config.Credential) { + t.Fatal("Control record did not open its credential", err) + } + selection, err := f.store.Selection(t.Context()) + if err != nil || selection.ID != config.ID || selection.TenantCapacityMiB != 4096 { + t.Fatal("selection", selection, err) + } + other := workspacepg.New(pgunit.NewPool(f.pool), f.cipher, uuid.NewString()) + if _, err := other.ActiveConfiguration(t.Context()); err == nil { + t.Fatal("credential opened for another installation") + } + if _, err := other.Selection(t.Context()); err != nil { + t.Fatal("public selection required the credential", err) + } + // The same ID is immutable, including its credential. + f.selectConfiguration(t, config, 4096) + for _, credential := range [][]byte{nil, []byte(`{"ssh_private_key":"another"}`)} { + changed := config + changed.Credential = credential + if err := f.execution.SelectConfiguration(admin(t), changed, 4096); !errors.Is(err, workspaces.ErrConflict) { + t.Fatal("credential of a retained configuration changed", err) + } + } + retained, err := f.store.Configuration(t.Context(), config.ID) + if err != nil || !bytes.Equal(retained.Credential, config.Credential) { + t.Fatal("retained configuration credential", err) + } + if _, err := f.store.Configuration(t.Context(), uuid.NewString()); !errors.Is(err, workspaces.ErrNotFound) { + t.Fatal("absent configuration", err) + } + if _, err := f.pool.Exec(t.Context(), `UPDATE workspace_fs_configurations SET credential=NULL WHERE id=$1`, config.ID); err == nil { + t.Fatal("database permitted credential mutation") + } +} + +func (f fixture) bindTenant(t *testing.T, tenant string, capacity uint32) (workspaces.Record, string, error) { + t.Helper() + session, environment := uuid.NewString(), uuid.NewString() + if _, err := f.pool.Exec(t.Context(), `INSERT INTO sessions(id, tenant_id, engine, idempotency_key, request_hash, configuration) + VALUES ($1, $2, 'codex', $3, 'hash', '{"environment":{"type":"openai_hosted"}}')`, session, tenant, uuid.NewString()); err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), `INSERT INTO environments(id, session_id) VALUES ($1, $2)`, environment, session); err != nil { + t.Fatal(err) + } + record, err := f.execution.Bind(t.Context(), tenant, environment, *f.active, capacity) + return record, session, err +} + +func TestTenantCapacityAdmission(t *testing.T) { + f := newFixture(t) + f.selectConfiguration(t, credentialed(), 7*1024) + tenant, other := uuid.NewString(), uuid.NewString() + var wg sync.WaitGroup + results := make(chan error, 10) + for range 10 { + wg.Go(func() { + _, _, err := f.bindTenant(t, tenant, 1024) + results <- err + }) + } + wg.Wait() + close(results) + admitted, refused := 0, 0 + for err := range results { + switch { + case err == nil: + admitted++ + case errors.Is(err, workspaces.ErrCapacityExhausted) && errors.Is(err, workspacefs.ErrUnavailable): + refused++ + default: + t.Fatal(err) + } + } + if admitted != 7 || refused != 3 { + t.Fatal("tenant capacity admitted", admitted, "refused", refused) + } + if _, _, err := f.bindTenant(t, other, 7*1024); err != nil { + t.Fatal("another tenant's capacity was consumed", err) + } + if _, _, err := f.bindTenant(t, other, 0); !errors.Is(err, workspacefs.ErrUnsupported) { + t.Fatal("unbounded workspace admitted under a tenant capacity", err) + } + + // A configuration switch keeps every unreleased capacity counted. + f.selectConfiguration(t, credentialed(), 7*1024) + if _, _, err := f.bindTenant(t, tenant, 1024); !errors.Is(err, workspaces.ErrCapacityExhausted) { + t.Fatal("configuration switch reset tenant usage", err) + } + var creating, ready workspaces.Record + var session string + if err := f.pool.QueryRow(t.Context(), `SELECT e.session_id FROM environment_workspaces w JOIN environments e ON e.id=w.environment_id JOIN sessions s ON s.id=e.session_id WHERE s.tenant_id=$1 LIMIT 1`, tenant).Scan(&session); err != nil { + t.Fatal(err) + } + var environment string + if err := f.pool.QueryRow(t.Context(), `SELECT id FROM environments WHERE session_id=$1`, session).Scan(&environment); err != nil { + t.Fatal(err) + } + creating, err := f.store.Get(t.Context(), tenant, environment) + if err != nil { + t.Fatal(err) + } + // Retained bindings bypass admission, including after a lower cap. + f.selectConfiguration(t, credentialed(), 1024) + if ready, err = f.rebind(t, creating); err != nil || ready.Reference != creating.Reference || ready.CapacityMiB != 1024 { + t.Fatal("retained binding blocked by a lower cap", err) + } + // Deleting still counts; physical release frees capacity. + f.selectConfiguration(t, credentialed(), 7*1024) + f.deleteSession(t, session) + if err := f.execution.BeginDelete(t.Context(), creating.Reference); err != nil { + t.Fatal(err) + } + if _, _, err := f.bindTenant(t, tenant, 1024); !errors.Is(err, workspaces.ErrCapacityExhausted) { + t.Fatal("deleting workspace released capacity before confirmation", err) + } + if err := f.execution.MarkDeleted(t.Context(), creating.Reference); err != nil { + t.Fatal(err) + } + if _, _, err := f.bindTenant(t, tenant, 1024); err != nil { + t.Fatal("confirmed release did not free capacity", err) + } + if _, err := f.pool.Exec(t.Context(), `UPDATE environment_workspaces SET capacity_mib=capacity_mib+1 WHERE object_id=$1`, creating.Reference.ObjectID); err == nil { + t.Fatal("database permitted admitted capacity mutation") + } +} + +func TestRetainedUnboundedWorkspaceBlocksTenantAdmission(t *testing.T) { + f := newFixture(t) + tenant, other := uuid.NewString(), uuid.NewString() + unbounded, _, err := f.bindTenant(t, tenant, 0) + if err != nil { + t.Fatal(err) + } + f.selectConfiguration(t, credentialed(), 8192) + if _, _, err := f.bindTenant(t, tenant, 1024); !errors.Is(err, workspaces.ErrCapacityExhausted) { + t.Fatal("tenant with an unbounded workspace admitted under a cap", err) + } + if _, _, err := f.bindTenant(t, other, 1024); err != nil { + t.Fatal("unrelated tenant blocked", err) + } + if replay, err := f.rebind(t, unbounded); err != nil || replay.Reference != unbounded.Reference || replay.CapacityMiB != 0 { + t.Fatal("existing unbounded workspace blocked or resized", err) + } +} + +func TestBindFencesConfigurationSwitch(t *testing.T) { + f := newFixture(t) + stale := *f.active + f.selectConfiguration(t, newConfiguration(), 0) + tenant, _, environment := f.environment(t) + if _, err := f.execution.Bind(t.Context(), tenant, environment, stale, 0); !errors.Is(err, workspaces.ErrConflict) { + t.Fatal("bound a configuration that is no longer selected", err) + } + if _, err := f.store.Get(t.Context(), tenant, environment); !errors.Is(err, workspaces.ErrNotFound) { + t.Fatal("conflicting bind left a row", err) + } + record, err := f.execution.Bind(t.Context(), tenant, environment, *f.active, 2048) + if err != nil || record.CapacityMiB != 2048 || record.Configuration.ID != *f.active { + t.Fatal(record, err) + } +} diff --git a/services/core/internal/persistence/postgres/workspacepg/store.go b/services/core/internal/persistence/postgres/workspacepg/store.go index 4109d2e2e..72de5ffb6 100644 --- a/services/core/internal/persistence/postgres/workspacepg/store.go +++ b/services/core/internal/persistence/postgres/workspacepg/store.go @@ -4,13 +4,16 @@ package workspacepg import ( "context" + "crypto/subtle" "encoding/json" "errors" + "fmt" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" @@ -18,45 +21,119 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspaces" ) -type Store struct{ pool *pgunit.Pool } +// credentials seals adapter credentials to this installation and their +// configuration. Records are opened only for Core's Control construction. +type credentials struct { + cipher *credentialcrypto.Cipher + installation string +} + +func (c credentials) seal(config workspacefs.ConfigurationRecord) ([]byte, error) { + if len(config.Credential) == 0 { + return nil, nil + } + return c.cipher.SealWorkspaceConfiguration(config.Credential, c.installation, config.ID) +} -func New(pool *pgunit.Pool) *Store { return &Store{pool: pool} } +// configuration opens a retained configuration. A credential that cannot be +// authenticated for this installation and configuration is an internal error. +func (c credentials) configuration(row sqlc.WorkspaceFsConfiguration) (workspacefs.ConfigurationRecord, error) { + record := workspacefs.ConfigurationRecord{Configuration: publicConfiguration(row)} + if len(row.Credential) > 0 { + var err error + if record.Credential, err = c.cipher.OpenWorkspaceConfiguration(row.Credential, c.installation, record.ID); err != nil { + return workspacefs.ConfigurationRecord{}, fmt.Errorf("workspace configuration credential: %w", err) + } + } + return record, nil +} + +type Store struct { + pool *pgunit.Pool + credentials credentials +} + +func New(pool *pgunit.Pool, cipher *credentialcrypto.Cipher, installation string) *Store { + return &Store{pool: pool, credentials: credentials{cipher, installation}} +} var _ workspaces.Storage = (*Store)(nil) -func (s *Store) ActiveConfiguration(ctx context.Context) (workspacefs.Configuration, error) { +func (s *Store) ActiveConfiguration(ctx context.Context) (workspacefs.ConfigurationRecord, error) { row, err := s.pool.Queries().GetActiveWorkspaceConfiguration(ctx) if errors.Is(err, pgx.ErrNoRows) { - return workspacefs.Configuration{}, workspaces.ErrNotConfigured + return workspacefs.ConfigurationRecord{}, workspaces.ErrNotConfigured } if err != nil { - return workspacefs.Configuration{}, err + return workspacefs.ConfigurationRecord{}, err } - return configuration(row), nil + return s.credentials.configuration(row) } -// SelectConfiguration atomically changes the selection without changing any -// configuration's definition or any existing Environment's binding. -func (e *Execution) SelectConfiguration(ctx context.Context, config workspacefs.Configuration) error { +func (s *Store) Configuration(ctx context.Context, id string) (workspacefs.ConfigurationRecord, error) { + parsed, err := pgunit.ParseID(id) + if err != nil { + return workspacefs.ConfigurationRecord{}, workspacefs.ErrInvalid + } + row, err := s.pool.Queries().GetWorkspaceConfiguration(ctx, parsed) + if errors.Is(err, pgx.ErrNoRows) { + return workspacefs.ConfigurationRecord{}, workspaces.ErrNotFound + } + if err != nil { + return workspacefs.ConfigurationRecord{}, err + } + return s.credentials.configuration(row) +} + +// Selection reads the public selection and policy in one statement; it never +// opens the credential. +func (s *Store) Selection(ctx context.Context) (workspaces.Selection, error) { + row, err := s.pool.Queries().GetWorkspaceSelection(ctx) + if errors.Is(err, pgx.ErrNoRows) { + return workspaces.Selection{}, workspaces.ErrNotConfigured + } + if err != nil { + return workspaces.Selection{}, err + } + return workspaces.Selection{Configuration: publicConfiguration(row.WorkspaceFsConfiguration), TenantCapacityMiB: uint32(row.TenantCapacityMib)}, nil +} + +// SelectConfiguration atomically changes the selection and tenant capacity +// policy without changing any configuration's definition or any existing +// Environment's binding. The admission row lock fences concurrent Bind calls. +func (e *Execution) SelectConfiguration(ctx context.Context, config workspacefs.ConfigurationRecord, tenantCapacityMiB uint32) error { if err := config.Validate(); err != nil { return err } + sealed, err := e.credentials.seal(config) + if err != nil { + return err + } id := pgunit.PathID(config.ID) return e.lease.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { q := sqlc.New(tx) if err := q.LockWorkspaceConfigurations(ctx); err != nil { return err } - if err := q.InsertWorkspaceConfiguration(ctx, sqlc.InsertWorkspaceConfigurationParams{ID: id, Adapter: config.Adapter, Parameters: config.Parameters}); err != nil { + if err := q.InsertWorkspaceConfiguration(ctx, sqlc.InsertWorkspaceConfigurationParams{ID: id, Adapter: config.Adapter, Parameters: config.Parameters, Credential: sealed}); err != nil { return err } - matches, err := q.MatchWorkspaceConfiguration(ctx, sqlc.MatchWorkspaceConfigurationParams{ID: id, Adapter: config.Adapter, Parameters: config.Parameters}) + match, err := q.MatchWorkspaceConfiguration(ctx, sqlc.MatchWorkspaceConfigurationParams{ID: id, Adapter: config.Adapter, Parameters: config.Parameters}) if err != nil { return err } - if !matches { + var stored []byte + if len(match.Credential) > 0 { + if stored, err = e.credentials.cipher.OpenWorkspaceConfiguration(match.Credential, e.credentials.installation, config.ID); err != nil { + return fmt.Errorf("workspace configuration credential: %w", err) + } + } + if !match.Matches || len(stored) != len(config.Credential) || subtle.ConstantTimeCompare(stored, config.Credential) != 1 { return workspaces.ErrConflict } + if err := q.SetWorkspaceTenantCapacity(ctx, int64(tenantCapacityMiB)); err != nil { + return err + } if err := q.ClearActiveWorkspaceConfiguration(ctx); err != nil { return err } @@ -68,7 +145,7 @@ func (e *Execution) SelectConfiguration(ctx context.Context, config workspacefs. } func (s *Store) Get(ctx context.Context, tenant, environment string) (workspaces.Record, error) { - return get(ctx, s.pool.Queries(), tenant, environment) + return get(ctx, s.pool.Queries(), s.credentials, tenant, environment) } func (s *Store) DeletionCandidates(ctx context.Context, after string) ([]workspaces.Record, error) { @@ -86,7 +163,7 @@ func (s *Store) DeletionCandidates(ctx context.Context, after string) ([]workspa } result := make([]workspaces.Record, 0, len(rows)) for _, row := range rows { - record, err := record(row.EnvironmentWorkspace, row.WorkspaceFsConfiguration, row.TenantID) + record, err := record(s.credentials, row.EnvironmentWorkspace, row.WorkspaceFsConfiguration, row.TenantID) if err != nil { return nil, err } @@ -95,15 +172,24 @@ func (s *Store) DeletionCandidates(ctx context.Context, after string) ([]workspa return result, nil } -type Execution struct{ lease *pgunit.Lease } +type Execution struct { + lease *pgunit.Lease + credentials credentials +} -func NewExecution(lease *pgunit.Lease) *Execution { return &Execution{lease: lease} } +func NewExecution(lease *pgunit.Lease, cipher *credentialcrypto.Cipher, installation string) *Execution { + return &Execution{lease: lease, credentials: credentials{cipher, installation}} +} var _ workspaces.ExecutionStorage = (*Execution)(nil) -func (e *Execution) Bind(ctx context.Context, tenant, environment string) (workspaces.Record, error) { +func (e *Execution) Bind(ctx context.Context, tenant, environment, configuration string, capacityMiB uint32) (workspaces.Record, error) { + expected, err := pgunit.ParseID(configuration) + if err != nil { + return workspaces.Record{}, workspacefs.ErrInvalid + } var result workspaces.Record - err := e.lease.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { + err = e.lease.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { q := sqlc.New(tx) deleted, err := lockSession(ctx, q, tenant, environment) if err != nil { @@ -112,7 +198,7 @@ func (e *Execution) Bind(ctx context.Context, tenant, environment string) (works if deleted { return workspaces.ErrConflict } - result, err = get(ctx, q, tenant, environment) + result, err = get(ctx, q, e.credentials, tenant, environment) if err == nil { if result.State == workspaces.Deleting || result.State == workspaces.Deleted { return workspaces.ErrConflict @@ -122,6 +208,12 @@ func (e *Execution) Bind(ctx context.Context, tenant, environment string) (works if !errors.Is(err, workspaces.ErrNotFound) { return err } + // Selection changes update this row, so the policy and the selected + // configuration below are read as one committed setting. + tenantCapacity, err := q.LockWorkspaceAdmission(ctx) + if err != nil { + return err + } config, err := q.GetActiveWorkspaceConfiguration(ctx) if errors.Is(err, pgx.ErrNoRows) { return workspaces.ErrNotConfigured @@ -129,12 +221,20 @@ func (e *Execution) Bind(ctx context.Context, tenant, environment string) (works if err != nil { return err } + if config.ID != expected { + return workspaces.ErrConflict + } + if tenantCapacity > 0 { + if err := admitTenant(ctx, q, tenant, tenantCapacity, capacityMiB); err != nil { + return err + } + } if err := q.InsertEnvironmentWorkspace(ctx, sqlc.InsertEnvironmentWorkspaceParams{ - ObjectID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, EnvironmentID: pgunit.PathID(environment), ConfigurationID: config.ID, + ObjectID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, EnvironmentID: pgunit.PathID(environment), ConfigurationID: config.ID, CapacityMib: int64(capacityMiB), }); err != nil { return err } - result, err = get(ctx, q, tenant, environment) + result, err = get(ctx, q, e.credentials, tenant, environment) return err }) if err != nil { @@ -143,12 +243,33 @@ func (e *Execution) Bind(ctx context.Context, tenant, environment string) (works return result, nil } +// admitTenant serializes one tenant's admissions and counts every workspace +// whose storage is not yet confirmed released. A retained unbounded workspace +// has no admitted capacity to count, so it blocks further admission. +func admitTenant(ctx context.Context, q *sqlc.Queries, tenant string, limit int64, capacityMiB uint32) error { + if capacityMiB == 0 { + return fmt.Errorf("%w: tenant capacity admission requires a bounded workspace", workspacefs.ErrUnsupported) + } + id := pgunit.PathID(tenant) + if err := q.LockWorkspaceTenant(ctx, id); err != nil { + return err + } + usage, err := q.TenantWorkspaceCapacity(ctx, id) + if err != nil { + return err + } + if usage.Unbounded || usage.CommittedMib+int64(capacityMiB) > limit { + return workspaces.ErrCapacityExhausted + } + return nil +} + func (e *Execution) MarkReady(ctx context.Context, ref workspacefs.Reference, attachment workspacefs.Attachment) error { return e.change(ctx, ref, func(ctx context.Context, q *sqlc.Queries, current workspaces.Record, deleted bool) error { if deleted || current.State != workspaces.Creating { return workspaces.ErrConflict } - if err := workspacefs.ValidateAttachment(ref, current.Configuration, attachment); err != nil { + if err := workspacefs.ValidateAttachment(ref, current.Configuration.Configuration, attachment); err != nil { return err } raw, err := json.Marshal(attachment) @@ -206,7 +327,7 @@ func (e *Execution) change(ctx context.Context, ref workspacefs.Reference, apply if err != nil { return err } - current, err := get(ctx, q, ref.TenantID, ref.EnvironmentID) + current, err := get(ctx, q, e.credentials, ref.TenantID, ref.EnvironmentID) if err != nil { return err } @@ -229,7 +350,7 @@ func lockSession(ctx context.Context, q *sqlc.Queries, tenant, environment strin return q.WorkspaceSessionDeleted(ctx, id) } -func get(ctx context.Context, q *sqlc.Queries, tenant, environment string) (workspaces.Record, error) { +func get(ctx context.Context, q *sqlc.Queries, c credentials, tenant, environment string) (workspaces.Record, error) { row, err := q.GetEnvironmentWorkspace(ctx, sqlc.GetEnvironmentWorkspaceParams{TenantID: pgunit.PathID(tenant), ID: pgunit.PathID(environment)}) if errors.Is(err, pgx.ErrNoRows) { return workspaces.Record{}, workspaces.ErrNotFound @@ -237,24 +358,28 @@ func get(ctx context.Context, q *sqlc.Queries, tenant, environment string) (work if err != nil { return workspaces.Record{}, err } - return record(row.EnvironmentWorkspace, row.WorkspaceFsConfiguration, row.TenantID) + return record(c, row.EnvironmentWorkspace, row.WorkspaceFsConfiguration, row.TenantID) } -func configuration(row sqlc.WorkspaceFsConfiguration) workspacefs.Configuration { +func publicConfiguration(row sqlc.WorkspaceFsConfiguration) workspacefs.Configuration { return workspacefs.Configuration{ID: uuid.UUID(row.ID.Bytes).String(), Adapter: row.Adapter, Parameters: row.Parameters} } -func record(row sqlc.EnvironmentWorkspace, config sqlc.WorkspaceFsConfiguration, tenant pgtype.UUID) (workspaces.Record, error) { +func record(c credentials, row sqlc.EnvironmentWorkspace, config sqlc.WorkspaceFsConfiguration, tenant pgtype.UUID) (workspaces.Record, error) { + configuration, err := c.configuration(config) + if err != nil { + return workspaces.Record{}, err + } result := workspaces.Record{ Reference: workspacefs.Reference{TenantID: uuid.UUID(tenant.Bytes).String(), EnvironmentID: uuid.UUID(row.EnvironmentID.Bytes).String(), ObjectID: uuid.UUID(row.ObjectID.Bytes).String()}, - Configuration: configuration(config), State: workspaces.State(row.State), + Configuration: configuration, CapacityMiB: uint32(row.CapacityMib), State: workspaces.State(row.State), } if row.Attachment != nil { var attachment workspacefs.Attachment if err := json.Unmarshal(row.Attachment, &attachment); err != nil { return workspaces.Record{}, err } - if err := workspacefs.ValidateAttachment(result.Reference, result.Configuration, attachment); err != nil { + if err := workspacefs.ValidateAttachment(result.Reference, result.Configuration.Configuration, attachment); err != nil { return workspaces.Record{}, err } result.Attachment = &attachment diff --git a/services/core/internal/persistence/postgres/workspacepg/store_test.go b/services/core/internal/persistence/postgres/workspacepg/store_test.go index 8d9d93532..cf093863d 100644 --- a/services/core/internal/persistence/postgres/workspacepg/store_test.go +++ b/services/core/internal/persistence/postgres/workspacepg/store_test.go @@ -1,6 +1,7 @@ package workspacepg_test import ( + "bytes" "context" "errors" "strings" @@ -12,6 +13,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/workspacepg" @@ -20,10 +22,13 @@ import ( ) type fixture struct { - pool *pgxpool.Pool - store *workspacepg.Store - execution *workspacepg.Execution - lease *pgunit.Lease + pool *pgxpool.Pool + store *workspacepg.Store + execution *workspacepg.Execution + lease *pgunit.Lease + cipher *credentialcrypto.Cipher + installation string + active *string } func newFixture(t *testing.T) fixture { @@ -34,15 +39,27 @@ func newFixture(t *testing.T) fixture { t.Fatal(err) } t.Cleanup(func() { _ = lease.Close(context.Background()) }) - f := fixture{pool: pool, store: workspacepg.New(pgunit.NewPool(pool)), execution: workspacepg.NewExecution(lease), lease: lease} - if err := f.execution.SelectConfiguration(admin(t), newConfiguration()); err != nil { + cipher, err := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) + if err != nil { t.Fatal(err) } + installation := uuid.NewString() + f := fixture{pool: pool, store: workspacepg.New(pgunit.NewPool(pool), cipher, installation), execution: workspacepg.NewExecution(lease, cipher, installation), lease: lease, + cipher: cipher, installation: installation, active: new(string)} + f.selectConfiguration(t, newConfiguration(), 0) return f } -func newConfiguration() workspacefs.Configuration { - return workspacefs.Configuration{ID: uuid.NewString(), Adapter: "fixture", Parameters: []byte(`{"storage":"test"}`)} +func newConfiguration() workspacefs.ConfigurationRecord { + return workspacefs.ConfigurationRecord{Configuration: workspacefs.Configuration{ID: uuid.NewString(), Adapter: "fixture", Parameters: []byte(`{"storage":"test"}`)}} +} + +func (f fixture) selectConfiguration(t *testing.T, config workspacefs.ConfigurationRecord, tenantCapacity uint32) { + t.Helper() + if err := f.execution.SelectConfiguration(admin(t), config, tenantCapacity); err != nil { + t.Fatal(err) + } + *f.active = config.ID } func admin(t *testing.T) context.Context { @@ -57,8 +74,9 @@ func TestConfigurationSelectionRequiresAtomicAudit(t *testing.T) { t.Fatal(err) } next := newConfiguration() - next.Parameters = []byte(`{"secret":"private-configuration-value"}`) - if err := f.execution.SelectConfiguration(t.Context(), next); !errors.Is(err, adminaudit.ErrInvalidSource) { + next.Parameters = []byte(`{"public":"configuration-value"}`) + next.Credential = []byte(`{"key":"private-configuration-value"}`) + if err := f.execution.SelectConfiguration(t.Context(), next, 0); !errors.Is(err, adminaudit.ErrInvalidSource) { t.Fatal("selection without audit source", err) } active, err := f.store.ActiveConfiguration(t.Context()) @@ -74,7 +92,7 @@ func TestConfigurationSelectionRequiresAtomicAudit(t *testing.T) { } ctx := admin(t) for range 2 { - if err := f.execution.SelectConfiguration(ctx, next); err != nil { + if err := f.execution.SelectConfiguration(ctx, next, 0); err != nil { t.Fatal("immutable selection retry", err) } } @@ -116,13 +134,18 @@ func (f fixture) environment(t *testing.T) (string, string, string) { func (f fixture) bind(t *testing.T) (workspaces.Record, string) { t.Helper() tenant, session, environment := f.environment(t) - record, err := f.execution.Bind(t.Context(), tenant, environment) + record, err := f.execution.Bind(t.Context(), tenant, environment, *f.active, 0) if err != nil { t.Fatal(err) } return record, session } +func (f fixture) rebind(t *testing.T, record workspaces.Record) (workspaces.Record, error) { + t.Helper() + return f.execution.Bind(t.Context(), record.Reference.TenantID, record.Reference.EnvironmentID, record.Configuration.ID, record.CapacityMiB) +} + func attachment(record workspaces.Record) workspacefs.Attachment { return workspacefs.Attachment{Reference: record.Reference, ConfigurationID: record.Configuration.ID, Kind: workspacefs.AttachmentHostDirectory, Native: []byte(`{"receipt":"fixture"}`)} @@ -139,10 +162,8 @@ func TestConfigurationSwitchPreservesBinding(t *testing.T) { f := newFixture(t) first, _ := f.bind(t) second := newConfiguration() - if err := f.execution.SelectConfiguration(admin(t), second); err != nil { - t.Fatal(err) - } - replayed, err := f.execution.Bind(t.Context(), first.Reference.TenantID, first.Reference.EnvironmentID) + f.selectConfiguration(t, second, 0) + replayed, err := f.execution.Bind(t.Context(), first.Reference.TenantID, first.Reference.EnvironmentID, second.ID, 0) if err != nil || replayed.Reference != first.Reference || replayed.Configuration.ID != first.Configuration.ID { t.Fatal("configuration switch changed existing creation", replayed, err) } @@ -152,14 +173,14 @@ func TestConfigurationSwitchPreservesBinding(t *testing.T) { } changed := first.Configuration changed.Parameters = []byte(`{"storage":"other"}`) - if err := f.execution.SelectConfiguration(admin(t), changed); !errors.Is(err, workspaces.ErrConflict) { + if err := f.execution.SelectConfiguration(admin(t), changed, 0); !errors.Is(err, workspaces.ErrConflict) { t.Fatal("mutated configuration", err) } active, err := f.store.ActiveConfiguration(t.Context()) if err != nil || active.ID != second.ID { t.Fatal("rejected selection changed active configuration", active, err) } - if err := f.execution.SelectConfiguration(admin(t), first.Configuration); err != nil { + if err := f.execution.SelectConfiguration(admin(t), first.Configuration, 0); err != nil { t.Fatal("could not reselect retained configuration", err) } if _, err := f.pool.Exec(t.Context(), `UPDATE workspace_fs_configurations SET adapter='other' WHERE id=$1`, first.Configuration.ID); err == nil { @@ -175,7 +196,7 @@ func TestConcurrentSelectionHasOneActiveConfiguration(t *testing.T) { var wg sync.WaitGroup errors := make(chan error, 8) for range 8 { - wg.Go(func() { errors <- f.execution.SelectConfiguration(admin(t), newConfiguration()) }) + wg.Go(func() { errors <- f.execution.SelectConfiguration(admin(t), newConfiguration(), 0) }) } wg.Wait() close(errors) @@ -249,7 +270,7 @@ func TestDeletedSessionRejectsLateReadyAndQueuesWithoutAllocation(t *testing.T) if err != nil || retained.State != workspaces.Deleted || retained.Reference != record.Reference { t.Fatal("deleted identity not retained", retained, err) } - if _, err := f.execution.Bind(t.Context(), record.Reference.TenantID, record.Reference.EnvironmentID); !errors.Is(err, workspaces.ErrConflict) { + if _, err := f.rebind(t, record); !errors.Is(err, workspaces.ErrConflict) { t.Fatal("deleted identity reused", err) } if err := f.execution.MarkReady(t.Context(), record.Reference, attachment(record)); !errors.Is(err, workspaces.ErrConflict) { @@ -345,7 +366,7 @@ func TestFailedCreationRetainedUntilExplicitSessionDeletion(t *testing.T) { if err := f.execution.MarkDeleted(t.Context(), record.Reference); err != nil { t.Fatal(err) } - if _, err := f.execution.Bind(t.Context(), record.Reference.TenantID, record.Reference.EnvironmentID); !errors.Is(err, workspaces.ErrConflict) { + if _, err := f.rebind(t, record); !errors.Is(err, workspaces.ErrConflict) { t.Fatal("deleted identity reused", err) } } @@ -358,7 +379,7 @@ func TestTenantAndAttachmentOwnership(t *testing.T) { if _, err := f.store.Get(t.Context(), foreign.TenantID, foreign.EnvironmentID); !errors.Is(err, workspaces.ErrNotFound) { t.Fatal("foreign tenant read workspace", err) } - if _, err := f.execution.Bind(t.Context(), foreign.TenantID, foreign.EnvironmentID); !errors.Is(err, workspaces.ErrNotFound) { + if _, err := f.execution.Bind(t.Context(), foreign.TenantID, foreign.EnvironmentID, record.Configuration.ID, 0); !errors.Is(err, workspaces.ErrNotFound) { t.Fatal("foreign tenant rebound workspace", err) } if err := f.execution.MarkReady(t.Context(), foreign, attachment(record)); !errors.Is(err, workspaces.ErrNotFound) { @@ -394,9 +415,9 @@ func TestWorkspaceWritesRequireExecutionLease(t *testing.T) { t.Fatal(err) } for name, write := range map[string]func() error{ - "configuration": func() error { return f.execution.SelectConfiguration(admin(t), newConfiguration()) }, + "configuration": func() error { return f.execution.SelectConfiguration(admin(t), newConfiguration(), 0) }, "bind": func() error { - _, err := f.execution.Bind(t.Context(), record.Reference.TenantID, record.Reference.EnvironmentID) + _, err := f.rebind(t, record) return err }, "ready": func() error { return f.execution.MarkReady(t.Context(), record.Reference, attachment(record)) }, diff --git a/services/core/internal/workspacefs/nfs/nfs.go b/services/core/internal/workspacefs/nfs/nfs.go index c01453ffe..11a397a5f 100644 --- a/services/core/internal/workspacefs/nfs/nfs.go +++ b/services/core/internal/workspacefs/nfs/nfs.go @@ -17,9 +17,10 @@ import ( const Name = "nfs" type parameters struct { - Root string `json:"root"` - NamespaceID string `json:"namespace_id"` - UID uint32 `json:"uid"` + Root string `json:"root"` + NamespaceID string `json:"namespace_id"` + UID uint32 `json:"uid"` + Quota *quotaParameters `json:"quota,omitempty"` } // CanonicalParameters rejects unknown, duplicate, missing and malformed settings. @@ -32,6 +33,15 @@ func CanonicalParameters(raw json.RawMessage) (json.RawMessage, error) { if err != nil || id == uuid.Nil || id.String() != p.NamespaceID || p.UID == 0 || p.UID > 2147483647 || !filepath.IsAbs(p.Root) || filepath.Clean(p.Root) != p.Root || bytes.IndexByte([]byte(p.Root), 0) >= 0 { return nil, fmt.Errorf("%w: root, namespace_id or uid", workspacefs.ErrInvalid) } + if p.Quota != nil { + var fields map[string]json.RawMessage + if err = json.Unmarshal(raw, &fields); err != nil { + return nil, fmt.Errorf("%w: parameters", workspacefs.ErrInvalid) + } + if p.Quota, err = canonicalQuota(fields["quota"]); err != nil { + return nil, err + } + } return json.Marshal(p) } @@ -90,13 +100,55 @@ func strict(raw []byte, dst any) error { return nil } -// Adapter holds immutable configuration only; construction performs no filesystem I/O. -type Adapter struct { +// Registration implements every workspacefs.Adapter role for kernel NFS. +type Registration struct{} + +func (Registration) Normalize(raw, credential json.RawMessage) (json.RawMessage, []byte, error) { + canonical, err := CanonicalParameters(raw) + if err != nil { + return nil, nil, err + } + var p parameters + _ = json.Unmarshal(canonical, &p) + encoded, err := encodeCredential(p.Quota, credential) + if err != nil { + return nil, nil, err + } + return canonical, encoded, nil +} + +// Control requires exactly the credential its parameters declare. +func (Registration) Control(record workspacefs.ConfigurationRecord) (workspacefs.Control, error) { + f, err := newFilesystem(record.Configuration) + if err != nil { + return nil, err + } + if f.parameters.Quota == nil { + if len(record.Credential) > 0 { + return nil, fmt.Errorf("%w: credential without quota parameters", workspacefs.ErrInvalid) + } + return f, nil + } + if f.quota, err = newQuotaClient(*f.parameters.Quota, record.Credential); err != nil { + return nil, err + } + return f, nil +} + +// Resolver never receives a credential; it validates bindings and resolves paths. +func (Registration) Resolver(configuration workspacefs.Configuration) (workspacefs.Resolver, error) { + return newFilesystem(configuration) +} + +// Filesystem holds immutable configuration only; construction performs no I/O. +// quota is set only for a Control constructed with the configured credential. +type Filesystem struct { configuration workspacefs.Configuration parameters parameters + quota *quotaClient } -func New(configuration workspacefs.Configuration) (*Adapter, error) { +func newFilesystem(configuration workspacefs.Configuration) (*Filesystem, error) { if err := configuration.Validate(); err != nil { return nil, err } @@ -108,14 +160,18 @@ func New(configuration workspacefs.Configuration) (*Adapter, error) { return nil, err } configuration.Parameters = raw - a := &Adapter{configuration: configuration} - _ = json.Unmarshal(raw, &a.parameters) - return a, nil + f := &Filesystem{configuration: configuration} + _ = json.Unmarshal(raw, &f.parameters) + return f, nil } -func (*Adapter) Declaration() workspacefs.Declaration { - return workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true} + +func (f *Filesystem) Declaration() workspacefs.Declaration { + return workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true, CapacityQuota: f.parameters.Quota != nil} } +func unavailable(err error) error { return fmt.Errorf("%w: %w", workspacefs.ErrUnavailable, err) } +func unconfirmed(err error) error { return fmt.Errorf("%w: %w", workspacefs.ErrUnconfirmed, err) } + // Slots remain occupied until the kernel operation actually returns, even after timeout. var operations = make(chan struct{}, 32) @@ -147,11 +203,11 @@ type receipt struct { NamespaceID string `json:"namespace_id"` } -func (a *Adapter) attachment(ref workspacefs.Reference) workspacefs.Attachment { - raw, _ := json.Marshal(receipt{a.parameters.NamespaceID}) - return workspacefs.Attachment{Reference: ref, ConfigurationID: a.configuration.ID, Kind: workspacefs.AttachmentHostDirectory, Native: raw} +func (f *Filesystem) attachment(ref workspacefs.Reference) workspacefs.Attachment { + raw, _ := json.Marshal(receipt{f.parameters.NamespaceID}) + return workspacefs.Attachment{Reference: ref, ConfigurationID: f.configuration.ID, Kind: workspacefs.AttachmentHostDirectory, Native: raw} } -func (a *Adapter) Resolve(ctx context.Context, b workspacefs.Binding) (workspacefs.Directory, error) { +func (f *Filesystem) Resolve(ctx context.Context, b workspacefs.Binding) (workspacefs.Directory, error) { if err := b.Validate(); err != nil { return workspacefs.Directory{}, err } @@ -160,12 +216,12 @@ func (a *Adapter) Resolve(ctx context.Context, b workspacefs.Binding) (workspace return workspacefs.Directory{}, err } var native receipt - if b.Configuration.ID != a.configuration.ID || b.Configuration.Adapter != Name || !bytes.Equal(raw, a.configuration.Parameters) || strict(b.Attachment.Native, &native) != nil || native.NamespaceID != a.parameters.NamespaceID { + if b.Configuration.ID != f.configuration.ID || b.Configuration.Adapter != Name || !bytes.Equal(raw, f.configuration.Parameters) || strict(b.Attachment.Native, &native) != nil || native.NamespaceID != f.parameters.NamespaceID { return workspacefs.Directory{}, workspacefs.ErrOwnership } - _, err = a.Observe(ctx, b.Attachment.Reference) + _, err = f.Observe(ctx, b.Attachment.Reference) if err != nil { return workspacefs.Directory{}, err } - return workspacefs.Directory{Path: filepath.Join(a.parameters.Root, "objects", b.Attachment.Reference.ObjectID, "live", "data")}, nil + return workspacefs.Directory{Path: filepath.Join(f.parameters.Root, "objects", b.Attachment.Reference.ObjectID, "live", "data")}, nil } diff --git a/services/core/internal/workspacefs/nfs/nfs_linux.go b/services/core/internal/workspacefs/nfs/nfs_linux.go index 0af18aedc..e77541654 100644 --- a/services/core/internal/workspacefs/nfs/nfs_linux.go +++ b/services/core/internal/workspacefs/nfs/nfs_linux.go @@ -19,12 +19,9 @@ import ( const rootMarker = ".oac-storage-root" -func unavailable(err error) error { return fmt.Errorf("%w: %w", workspacefs.ErrUnavailable, err) } -func unconfirmed(err error) error { return fmt.Errorf("%w: %w", workspacefs.ErrUnconfirmed, err) } - // open validates the pinned directory descriptor before any namespace mutation. // It runs inside the process budget, including OpenRoot and every stat/read. -func (a *Adapter) open() (*os.Root, error) { +func (a *Filesystem) open() (*os.Root, error) { r, err := os.OpenRoot(a.parameters.Root) if err != nil { return nil, unavailable(err) @@ -120,7 +117,7 @@ func write(r *os.Root, name string, raw []byte) error { } return closeErr } -func (a *Adapter) Check(ctx context.Context) error { +func (a *Filesystem) Check(ctx context.Context) error { _, err := run(ctx, func() (struct{}, error) { r, err := a.open() if err != nil { @@ -151,6 +148,9 @@ func (a *Adapter) Check(ctx context.Context) error { if err = unix.Fremovexattr(fd, key); err != nil { return struct{}{}, unavailable(err) } + if a.quota != nil { + return struct{}{}, a.quota.check(ctx, a.parameters.NamespaceID) + } return struct{}{}, nil }) return err @@ -170,7 +170,7 @@ func identity(r *os.Root, name string, ref workspacefs.Reference) error { } // claim publishes an immutable synced identity with a no-replace hard link. -func (a *Adapter) claim(r *os.Root, ref workspacefs.Reference) error { +func (a *Filesystem) claim(r *os.Root, ref workspacefs.Reference) error { if err := mkdir(r, "objects", a.parameters.UID); err != nil { return err } @@ -248,7 +248,7 @@ func live(r *os.Root, ref workspacefs.Reference) error { } return nil } -func (a *Adapter) Observe(ctx context.Context, ref workspacefs.Reference) (workspacefs.Attachment, error) { +func (a *Filesystem) Observe(ctx context.Context, ref workspacefs.Reference) (workspacefs.Attachment, error) { if err := ref.Validate(); err != nil { return workspacefs.Attachment{}, err } @@ -261,7 +261,7 @@ func (a *Adapter) Observe(ctx context.Context, ref workspacefs.Reference) (works return a.observe(r, ref) }) } -func (a *Adapter) observe(r *os.Root, ref workspacefs.Reference) (workspacefs.Attachment, error) { +func (a *Filesystem) observe(r *os.Root, ref workspacefs.Reference) (workspacefs.Attachment, error) { if err := live(r, ref); err != nil { if errors.Is(err, fs.ErrNotExist) { err = workspacefs.ErrNotFound @@ -277,23 +277,36 @@ func (a *Adapter) observe(r *os.Root, ref workspacefs.Reference) (workspacefs.At return a.attachment(ref), nil } -func (a *Adapter) Create(ctx context.Context, ref workspacefs.Reference) (workspacefs.Attachment, error) { +func (a *Filesystem) Create(ctx context.Context, ref workspacefs.Reference, capacityMiB uint32) (workspacefs.Attachment, error) { if err := ref.Validate(); err != nil { return workspacefs.Attachment{}, err } + if capacityMiB > 0 && a.quota == nil { + return workspacefs.Attachment{}, fmt.Errorf("%w: capacity quota is not enforced", workspacefs.ErrUnsupported) + } return run(ctx, func() (workspacefs.Attachment, error) { r, err := a.open() if err != nil { return workspacefs.Attachment{}, err } defer r.Close() - if err = a.create(r, ref); err != nil { + var bound func() error + if capacityMiB > 0 { + bound = func() error { + return a.quota.ensure(ctx, ref, a.configuration.ID, a.parameters.NamespaceID, capacityMiB) + } + } + if err = a.create(r, ref, bound); err != nil { return workspacefs.Attachment{}, err } return a.attachment(ref), nil }) } -func (a *Adapter) create(r *os.Root, ref workspacefs.Reference) error { + +// create calls bound, when set, after staging exists and before any stage is +// created or an existing live object is returned, so the server binds the +// project to an empty staging root and every retry re-verifies the bound. +func (a *Filesystem) create(r *os.Root, ref workspacefs.Reference, bound func() error) error { if err := a.claim(r, ref); err != nil { return unconfirmed(err) } @@ -305,16 +318,24 @@ func (a *Adapter) create(r *os.Root, ref workspacefs.Reference) error { if dead { return workspacefs.ErrNotFound } - if err = live(r, ref); err == nil { - return nil - } else if !errors.Is(err, fs.ErrNotExist) { - return err - } for _, dir := range []string{"staging", "trash"} { if err = mkdir(r, path.Join(obj, dir), a.parameters.UID); err != nil { return unconfirmed(err) } } + if bound != nil { + if err = syncDir(r, obj); err != nil { + return unconfirmed(err) + } + if err = bound(); err != nil { + return err + } + } + if err = live(r, ref); err == nil { + return nil + } else if !errors.Is(err, fs.ErrNotExist) { + return err + } stage := path.Join(obj, "staging", uuid.NewString()) if err = r.Mkdir(stage, 0700); err != nil { return unconfirmed(err) @@ -390,7 +411,7 @@ func removeEnvelope(r *os.Root, dir string, ref workspacefs.Reference, recursive } return nil } -func (a *Adapter) Delete(ctx context.Context, ref workspacefs.Reference) error { +func (a *Filesystem) Delete(ctx context.Context, ref workspacefs.Reference) error { if err := ref.Validate(); err != nil { return err } @@ -405,7 +426,7 @@ func (a *Adapter) Delete(ctx context.Context, ref workspacefs.Reference) error { }) return err } -func (a *Adapter) delete(r *os.Root, ref workspacefs.Reference) error { +func (a *Filesystem) delete(r *os.Root, ref workspacefs.Reference) error { if err := a.claim(r, ref); err != nil { return unconfirmed(err) } diff --git a/services/core/internal/workspacefs/nfs/nfs_linux_test.go b/services/core/internal/workspacefs/nfs/nfs_linux_test.go index 4d485e380..92523c4f1 100644 --- a/services/core/internal/workspacefs/nfs/nfs_linux_test.go +++ b/services/core/internal/workspacefs/nfs/nfs_linux_test.go @@ -14,7 +14,7 @@ import ( "github.com/google/uuid" ) -func local(t *testing.T) (*Adapter, *os.Root, workspacefs.Reference) { +func local(t *testing.T) (*Filesystem, *os.Root, workspacefs.Reference) { t.Helper() dir := t.TempDir() r, err := os.OpenRoot(dir) @@ -25,14 +25,14 @@ func local(t *testing.T) (*Adapter, *os.Root, workspacefs.Reference) { if err = r.Mkdir("objects", 0700); err != nil { t.Fatal(err) } - return &Adapter{parameters: parameters{UID: uint32(os.Geteuid())}}, r, workspacefs.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), ObjectID: uuid.NewString()} + return &Filesystem{parameters: parameters{UID: uint32(os.Geteuid())}}, r, workspacefs.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), ObjectID: uuid.NewString()} } func TestLifecycleConcurrentReplay(t *testing.T) { a, r, ref := local(t) var wg sync.WaitGroup errs := make(chan error, 16) for range 16 { - wg.Go(func() { errs <- a.create(r, ref) }) + wg.Go(func() { errs <- a.create(r, ref, nil) }) } wg.Wait() close(errs) @@ -45,7 +45,7 @@ func TestLifecycleConcurrentReplay(t *testing.T) { if err := write(r, data, []byte("retain")); err != nil { t.Fatal(err) } - if err := a.create(r, ref); err != nil { + if err := a.create(r, ref, nil); err != nil { t.Fatal(err) } if raw, err := r.ReadFile(data); err != nil || string(raw) != "retain" { @@ -62,7 +62,7 @@ func TestLifecycleConcurrentReplay(t *testing.T) { t.Fatal(err) } } - if err := a.create(r, ref); !errors.Is(err, workspacefs.ErrNotFound) { + if err := a.create(r, ref, nil); !errors.Is(err, workspacefs.ErrNotFound) { t.Fatalf("retired create: %v", err) } if err := live(r, ref); !errors.Is(err, workspacefs.ErrNotFound) { @@ -74,7 +74,7 @@ func TestLifecycleConcurrentReplay(t *testing.T) { } func TestForeignIdentityNeverDeletes(t *testing.T) { a, r, ref := local(t) - if err := a.create(r, ref); err != nil { + if err := a.create(r, ref, nil); err != nil { t.Fatal(err) } other := ref @@ -88,7 +88,7 @@ func TestForeignIdentityNeverDeletes(t *testing.T) { } func TestMissingMarkerDoesNotDeleteData(t *testing.T) { a, r, ref := local(t) - if err := a.create(r, ref); err != nil { + if err := a.create(r, ref, nil); err != nil { t.Fatal(err) } dir := path.Join(object(ref), "live") @@ -106,7 +106,7 @@ func TestMissingMarkerDoesNotDeleteData(t *testing.T) { } } func TestCheckRejectsLocalFilesystem(t *testing.T) { - a := &Adapter{parameters: parameters{Root: t.TempDir(), UID: 65532}} + a := &Filesystem{parameters: parameters{Root: t.TempDir(), UID: 65532}} if err := a.Check(context.Background()); !errors.Is(err, workspacefs.ErrUnavailable) { t.Fatalf("local path accepted: %v", err) } @@ -120,7 +120,7 @@ func TestCreateDoesNotReplaceUnownedLive(t *testing.T) { if err := r.Mkdir(path.Join(object(ref), "live"), 0700); err != nil { t.Fatal(err) } - if err := a.create(r, ref); !errors.Is(err, workspacefs.ErrOwnership) { + if err := a.create(r, ref, nil); !errors.Is(err, workspacefs.ErrOwnership) { t.Fatalf("unowned live: %v", err) } } @@ -151,7 +151,51 @@ func TestObserveAbsentObjectsDoesNotPrepareNamespace(t *testing.T) { if err := a.delete(r, ref); err != nil { t.Fatal(err) } - if err := a.create(r, ref); !errors.Is(err, workspacefs.ErrNotFound) { + if err := a.create(r, ref, nil); !errors.Is(err, workspacefs.ErrNotFound) { t.Fatalf("deleted identity reused: %v", err) } } + +func TestQuotaBoundPrecedesStageAndEveryRetry(t *testing.T) { + a, r, ref := local(t) + calls := 0 + failure := workspacefs.ErrUnconfirmed + bound := func() error { + calls++ + entries, err := os.ReadDir(path.Join(r.Name(), object(ref), "staging")) + if err != nil || len(entries) != 0 { + t.Fatalf("bound after a stage was created: %v %v", entries, err) + } + if _, err := r.Lstat(path.Join(object(ref), "live")); err == nil && calls == 1 { + t.Fatal("bound after publication") + } + return failure + } + if err := a.create(r, ref, bound); !errors.Is(err, workspacefs.ErrUnconfirmed) || calls != 1 { + t.Fatalf("unbounded creation continued: %v", err) + } + if _, err := r.Lstat(path.Join(object(ref), "live")); !errors.Is(err, os.ErrNotExist) { + t.Fatal("published without a verified bound", err) + } + failure = nil + if err := a.create(r, ref, bound); err != nil || calls != 2 { + t.Fatal(err) + } + if err := a.create(r, ref, bound); err != nil || calls != 3 { + t.Fatal("published object retry skipped verification", err) + } + if err := a.delete(r, ref); err != nil { + t.Fatal(err) + } + if err := a.create(r, ref, bound); !errors.Is(err, workspacefs.ErrNotFound) || calls != 3 { + t.Fatal("deleted object reached the quota server", err) + } +} + +func TestCreateRejectsCapacityWithoutQuota(t *testing.T) { + a := &Filesystem{parameters: parameters{Root: t.TempDir(), UID: 65532}} + ref := workspacefs.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), ObjectID: uuid.NewString()} + if _, err := a.Create(context.Background(), ref, 1024); !errors.Is(err, workspacefs.ErrUnsupported) { + t.Fatal(err) + } +} diff --git a/services/core/internal/workspacefs/nfs/nfs_other.go b/services/core/internal/workspacefs/nfs/nfs_other.go index fc865b7c0..fd9474ff3 100644 --- a/services/core/internal/workspacefs/nfs/nfs_other.go +++ b/services/core/internal/workspacefs/nfs/nfs_other.go @@ -7,13 +7,13 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" ) -func (*Adapter) Check(context.Context) error { return workspacefs.ErrUnsupported } -func (*Adapter) Create(context.Context, workspacefs.Reference) (workspacefs.Attachment, error) { +func (*Filesystem) Check(context.Context) error { return workspacefs.ErrUnsupported } +func (*Filesystem) Create(context.Context, workspacefs.Reference, uint32) (workspacefs.Attachment, error) { return workspacefs.Attachment{}, workspacefs.ErrUnsupported } -func (*Adapter) Observe(context.Context, workspacefs.Reference) (workspacefs.Attachment, error) { +func (*Filesystem) Observe(context.Context, workspacefs.Reference) (workspacefs.Attachment, error) { return workspacefs.Attachment{}, workspacefs.ErrUnsupported } -func (*Adapter) Delete(context.Context, workspacefs.Reference) error { +func (*Filesystem) Delete(context.Context, workspacefs.Reference) error { return workspacefs.ErrUnsupported } diff --git a/services/core/internal/workspacefs/nfs/quota.go b/services/core/internal/workspacefs/nfs/quota.go new file mode 100644 index 000000000..a3b4c1046 --- /dev/null +++ b/services/core/internal/workspacefs/nfs/quota.go @@ -0,0 +1,238 @@ +package nfs + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "regexp" + "strconv" + "strings" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" + "github.com/google/uuid" + "golang.org/x/crypto/ssh" +) + +// The server command oac-workspace-quota (quota/oac-workspace-quota in this +// package) enforces XFS project quotas on the storage host. Its request, +// response and error codes are fixed by testdata/quota, shared by both tests. +const ( + inodesPerMiB = 256 + quotaCallLimit = 30 * time.Second + quotaOutputSize = 64 << 10 +) + +type quotaParameters struct { + Endpoint string `json:"endpoint"` + User string `json:"user"` + HostKey string `json:"host_key"` +} + +type quotaCredential struct { + SSHPrivateKey string `json:"ssh_private_key"` +} + +var quotaUser = regexp.MustCompile(`^[a-z_][a-z0-9_-]{0,31}$`) + +// canonicalQuota validates the public quota parameters. The pinned host key is +// stored in authorized-key form; a comment is dropped and options are refused. +func canonicalQuota(raw json.RawMessage) (*quotaParameters, error) { + invalid := fmt.Errorf("%w: quota endpoint, user or host_key", workspacefs.ErrInvalid) + var q quotaParameters + if strict(raw, &q) != nil || !quotaUser.MatchString(q.User) { + return nil, invalid + } + host, port, err := net.SplitHostPort(q.Endpoint) + number, portErr := strconv.ParseUint(port, 10, 16) + if err != nil || portErr != nil || number == 0 || host == "" || strings.ContainsAny(q.Endpoint, " \t\r\n\x00") { + return nil, invalid + } + key, _, options, rest, err := ssh.ParseAuthorizedKey([]byte(q.HostKey)) + if err != nil || len(options) > 0 || len(rest) > 0 { + return nil, invalid + } + q.HostKey = strings.TrimSpace(string(ssh.MarshalAuthorizedKey(key))) + return &q, nil +} + +// encodeCredential returns the canonical private credential: exactly one +// unencrypted SSH private key when quota is configured, otherwise nothing. +func encodeCredential(quota *quotaParameters, raw json.RawMessage) ([]byte, error) { + if len(bytes.TrimSpace(raw)) == 0 { + if quota != nil { + return nil, fmt.Errorf("%w: quota requires credential", workspacefs.ErrInvalid) + } + return nil, nil + } + var c quotaCredential + if quota == nil || strict(raw, &c) != nil { + return nil, fmt.Errorf("%w: credential", workspacefs.ErrInvalid) + } + if _, err := ssh.ParsePrivateKey([]byte(c.SSHPrivateKey)); err != nil { + return nil, fmt.Errorf("%w: credential", workspacefs.ErrInvalid) + } + return json.Marshal(c) +} + +type quotaClient struct { + address string + config ssh.ClientConfig +} + +func newQuotaClient(p quotaParameters, credential []byte) (*quotaClient, error) { + var c quotaCredential + if strict(credential, &c) != nil { + return nil, fmt.Errorf("%w: credential", workspacefs.ErrInvalid) + } + signer, err := ssh.ParsePrivateKey([]byte(c.SSHPrivateKey)) + if err != nil { + return nil, fmt.Errorf("%w: credential", workspacefs.ErrInvalid) + } + hostKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(p.HostKey)) + if err != nil { + return nil, fmt.Errorf("%w: host_key", workspacefs.ErrInvalid) + } + algorithms := []string{hostKey.Type()} + if hostKey.Type() == ssh.KeyAlgoRSA { + algorithms = []string{ssh.KeyAlgoRSASHA512, ssh.KeyAlgoRSASHA256} + } + return "aClient{address: p.Endpoint, config: ssh.ClientConfig{ + User: p.User, Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: ssh.FixedHostKey(hostKey), HostKeyAlgorithms: algorithms, Timeout: quotaCallLimit, + }}, nil +} + +type quotaRequest struct { + Operation string `json:"operation"` + NamespaceID string `json:"namespace_id"` + TenantID string `json:"tenant_id,omitempty"` + EnvironmentID string `json:"environment_id,omitempty"` + ObjectID string `json:"object_id,omitempty"` + ConfigurationID string `json:"configuration_id,omitempty"` + CapacityMiB uint32 `json:"capacity_mib,omitempty"` + InodesHard uint64 `json:"inodes_hard,omitempty"` +} + +type quotaResponse struct { + Error string `json:"error,omitempty"` + FilesystemUUID string `json:"filesystem_uuid,omitempty"` + ProjectID uint32 `json:"project_id,omitempty"` + BytesHard uint64 `json:"bytes_hard,omitempty"` + InodesHard uint64 `json:"inodes_hard,omitempty"` +} + +// quotaErrors maps every helper error code to the protocol outcome. +var quotaErrors = map[string]error{ + "invalid_request": workspacefs.ErrInvalid, + "object_deleted": workspacefs.ErrNotFound, + "identity_mismatch": workspacefs.ErrOwnership, + "binding_mismatch": workspacefs.ErrOwnership, + "object_replaced": workspacefs.ErrOwnership, + "foreign_project": workspacefs.ErrOwnership, + "nonempty_object": workspacefs.ErrOwnership, + "quota_mismatch": workspacefs.ErrOwnership, + "unsafe_path": workspacefs.ErrOwnership, + "object_missing": workspacefs.ErrUnavailable, + "enforcement_off": workspacefs.ErrUnavailable, + "range_exhausted": workspacefs.ErrUnavailable, + "journal_mismatch": workspacefs.ErrUnavailable, + "operation_failed": workspacefs.ErrUnavailable, +} + +type boundedBuffer struct{ bytes.Buffer } + +func (b *boundedBuffer) Write(p []byte) (int, error) { + if b.Len()+len(p) > quotaOutputSize { + return 0, errors.New("helper output exceeds limit") + } + return b.Buffer.Write(p) +} + +// call runs one helper request. Failure before the request is sent proves no +// mutation and is unavailable; afterwards a missing or malformed answer leaves +// the outcome unknown, which unconfirmed reports for mutating requests. +func (q *quotaClient) call(ctx context.Context, request quotaRequest) (quotaResponse, error) { + raw, err := json.Marshal(request) + if err != nil { + return quotaResponse{}, err + } + unknown := workspacefs.ErrUnconfirmed + if request.Operation == "check" { + unknown = workspacefs.ErrUnavailable + } + ctx, cancel := context.WithTimeout(ctx, quotaCallLimit) + defer cancel() + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", q.address) + if err != nil { + return quotaResponse{}, unavailable(err) + } + stop := context.AfterFunc(ctx, func() { conn.Close() }) + defer stop() + defer conn.Close() + deadline, _ := ctx.Deadline() + _ = conn.SetDeadline(deadline) + config := q.config + sshConn, channels, requests, err := ssh.NewClientConn(conn, q.address, &config) + if err != nil { + return quotaResponse{}, unavailable(err) + } + client := ssh.NewClient(sshConn, channels, requests) + defer client.Close() + session, err := client.NewSession() + if err != nil { + return quotaResponse{}, unavailable(err) + } + defer session.Close() + var output boundedBuffer + session.Stdin, session.Stdout, session.Stderr = bytes.NewReader(raw), &output, io.Discard + // The forced command ignores the requested command; the verb is in the request. + runErr := session.Run("oac-workspace-quota") + var response quotaResponse + if strict(bytes.TrimSpace(output.Bytes()), &response) != nil { + return quotaResponse{}, fmt.Errorf("%w: helper response: %v", unknown, runErr) + } + if response.Error != "" { + mapped, ok := quotaErrors[response.Error] + if !ok { + return quotaResponse{}, fmt.Errorf("%w: helper error", unknown) + } + return quotaResponse{}, fmt.Errorf("%w: helper %s", mapped, response.Error) + } + if runErr != nil || !canonicalUUID(response.FilesystemUUID) { + return quotaResponse{}, fmt.Errorf("%w: helper response", unknown) + } + return response, nil +} + +func canonicalUUID(value string) bool { + id, err := uuid.Parse(value) + return err == nil && id != uuid.Nil && id.String() == value +} + +func (q *quotaClient) check(ctx context.Context, namespace string) error { + response, err := q.call(ctx, quotaRequest{Operation: "check", NamespaceID: namespace}) + if err == nil && (response.ProjectID != 0 || response.BytesHard != 0 || response.InodesHard != 0) { + err = fmt.Errorf("%w: helper check response", workspacefs.ErrUnavailable) + } + return err +} + +// ensure binds the object's staging tree to its own project with exactly the +// admitted byte and inode hard limits, verified by the server before success. +func (q *quotaClient) ensure(ctx context.Context, ref workspacefs.Reference, configuration, namespace string, capacityMiB uint32) error { + inodes := uint64(capacityMiB) * inodesPerMiB + response, err := q.call(ctx, quotaRequest{Operation: "ensure", NamespaceID: namespace, TenantID: ref.TenantID, EnvironmentID: ref.EnvironmentID, + ObjectID: ref.ObjectID, ConfigurationID: configuration, CapacityMiB: capacityMiB, InodesHard: inodes}) + if err != nil { + return err + } + if response.ProjectID == 0 || response.BytesHard != uint64(capacityMiB)<<20 || response.InodesHard != inodes { + return fmt.Errorf("%w: helper receipt differs from the admitted capacity", workspacefs.ErrOwnership) + } + return nil +} diff --git a/services/core/internal/workspacefs/nfs/quota/helper_test.py b/services/core/internal/workspacefs/nfs/quota/helper_test.py new file mode 100644 index 000000000..1b6fce7f8 --- /dev/null +++ b/services/core/internal/workspacefs/nfs/quota/helper_test.py @@ -0,0 +1,278 @@ +"""Unit tests for oac-workspace-quota with a fake XFS backend. + +Directory traversal, ownership, the journal and every state transition are +real; only privileged XFS ioctls and xfs_quota are replaced. Live qualification +on a real XFS filesystem is separate. +""" + +import importlib.machinery +import importlib.util +import io +import json +import os +import pathlib +import tempfile +import threading +import unittest + +HERE = pathlib.Path(__file__).resolve().parent +FIXTURES = HERE.parent / "testdata" / "quota" +loader = importlib.machinery.SourceFileLoader("oac_workspace_quota", str(HERE / "oac-workspace-quota")) +spec = importlib.util.spec_from_loader(loader.name, loader) +helper = importlib.util.module_from_spec(spec) +loader.exec_module(helper) + +ENSURE = json.loads((FIXTURES / "ensure-request.json").read_text()) +FILESYSTEM = json.loads((FIXTURES / "ensure-response.json").read_text())["filesystem_uuid"] + + +class Crash(Exception): + pass + + +class FakeXFS: + """In-memory project attributes and dquots keyed by real inode numbers.""" + attributes_by_inode = {} + generations = {} + limits_by_project = {} + usage_by_project = {} + enforcement = True + filesystem = FILESYSTEM + + def __init__(self, mount_point=None, lock_fd=None): + pass + + @classmethod + def reset(cls): + cls.attributes_by_inode, cls.generations, cls.limits_by_project, cls.usage_by_project = {}, {}, {}, {} + cls.enforcement, cls.filesystem = True, FILESYSTEM + + def filesystem_uuid(self, fd): + return self.filesystem + + def attributes(self, fd): + return self.attributes_by_inode.get(os.fstat(fd).st_ino, (0, False)) + + def project(self, fd, project): + self.attributes_by_inode[os.fstat(fd).st_ino] = (project, True) + self.usage_by_project[project] = (4096, 1) + + def generation(self, fd): + return self.generations.get(os.fstat(fd).st_ino, 1) + + def enforced(self): + return self.enforcement + + def limits(self, project): + return self.limits_by_project.get(project, (0, 0, 0, 0, 0, 0)) + + def usage(self, project): + return self.usage_by_project.get(project, (0, 0)) + + def set_limits(self, project, bytes_hard, inodes_hard): + self.limits_by_project[project] = (0, bytes_hard, 0, inodes_hard, 0, 0) + + +class HelperTest(unittest.TestCase): + def setUp(self): + FakeXFS.reset() + helper.BACKEND = FakeXFS + helper.step = lambda name: None + self.directory = tempfile.TemporaryDirectory() + base = pathlib.Path(self.directory.name) + self.mount = base / "xfs" + self.export = self.mount / "export" + self.export.mkdir(parents=True, mode=0o700) + os.chmod(self.export, 0o700) + marker = self.export / ".oac-storage-root" + marker.write_text(ENSURE["namespace_id"] + "\n") + os.chmod(marker, 0o600) + self.config = base / "config.json" + self.config.write_text(json.dumps({"export_root": str(self.export), "mount_point": str(self.mount)})) + os.chmod(self.config, 0o600) + self.assertEqual(helper.init(str(self.config), 1000, 1004)["projid_start"], 1000) + + def tearDown(self): + self.directory.cleanup() + + def envelope(self, request=ENSURE): + objects = self.export / "objects" + objects.mkdir(mode=0o700, exist_ok=True) + obj = objects / request["object_id"] + obj.mkdir(mode=0o700) + for name in ("staging", "trash"): + (obj / name).mkdir(mode=0o700) + identity = obj / "identity" + identity.write_text(json.dumps({k: request[k] for k in ("tenant_id", "environment_id", "object_id")})) + os.chmod(identity, 0o600) + return obj + + def serve(self, request): + return helper.serve(str(self.config), io.BytesIO(json.dumps(request).encode())) + + def other(self, **changes): + request = dict(ENSURE, object_id=str(__import__("uuid").uuid4())) + request.update(changes) + return request + + def test_fixture_round_trip_and_replay(self): + self.envelope() + expected = json.loads((FIXTURES / "ensure-response.json").read_text()) + for _ in range(3): + self.assertEqual(self.serve(ENSURE), expected) + result = helper.serve(str(self.config), io.BytesIO((FIXTURES / "ensure-request.json").read_bytes())) + self.assertEqual(json.dumps(result, sort_keys=True) + "\n", (FIXTURES / "ensure-response.json").read_text()) + check = helper.serve(str(self.config), io.BytesIO((FIXTURES / "check-request.json").read_bytes())) + self.assertEqual(json.dumps(check, sort_keys=True) + "\n", (FIXTURES / "check-response.json").read_text()) + + def test_error_codes_match_shared_fixture(self): + self.assertEqual(helper.ERROR_CODES, frozenset(json.loads((FIXTURES / "errors.json").read_text()))) + + def test_concurrent_same_reference_converges(self): + self.envelope() + results = [] + threads = [threading.Thread(target=lambda: results.append(self.serve(ENSURE))) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.assertEqual({r.get("project_id") for r in results}, {1000}) + + def test_distinct_references_never_share_ids_and_range_is_bounded(self): + projects = [] + for _ in range(5): + request = self.other() + self.envelope(request) + projects.append(self.serve(request)["project_id"]) + self.assertEqual(projects, [1000, 1001, 1002, 1003, 1004]) + request = self.other() + self.envelope(request) + self.assertEqual(self.serve(request), {"error": "range_exhausted"}) + check = self.serve({"operation": "check", "namespace_id": ENSURE["namespace_id"]}) + self.assertEqual(check, {"error": "range_exhausted"}) + + def test_crash_at_every_step_replays_same_project_and_limits(self): + for point in ("allocated", "projected-inode", "projected", "limited", "ready"): + FakeXFS.reset() + request = self.other() + self.envelope(request) + + def crash(name, point=point): + if name == point: + raise Crash(name) + helper.step = crash + with self.assertRaises(Crash): + self.serve(request) + helper.step = lambda name: None + first = self.serve(request) + self.assertNotIn("error", first, point) + self.assertEqual(self.serve(request), first, point) + self.assertEqual(FakeXFS.limits_by_project[first["project_id"]], (0, 2048 << 20, 0, 524288, 0, 0)) + + def test_binding_changes_are_refused(self): + self.envelope() + self.assertEqual(self.serve(ENSURE)["project_id"], 1000) + for change in ({"capacity_mib": 4096, "inodes_hard": 4096 * 256}, {"inodes_hard": 1}, + {"configuration_id": "77777777-7777-4777-8777-777777777777"}): + self.assertEqual(self.serve(dict(ENSURE, **change)), {"error": "binding_mismatch"}) + self.assertEqual(self.serve(dict(ENSURE, tenant_id="77777777-7777-4777-8777-777777777777")), {"error": "identity_mismatch"}) + + def test_replaced_staging_inode_is_refused(self): + obj = self.envelope() + self.serve(ENSURE) + (obj / "staging").rmdir() + (obj / "staging").mkdir(mode=0o700) + # A reused inode number has a new generation. + FakeXFS.generations[os.stat(obj / "staging").st_ino] = 2 + self.assertEqual(self.serve(ENSURE), {"error": "object_replaced"}) + + def test_foreign_or_inherited_projects_are_refused(self): + obj = self.envelope() + FakeXFS.attributes_by_inode[os.stat(obj.parent).st_ino] = (9, True) + self.assertEqual(self.serve(ENSURE), {"error": "foreign_project"}) + FakeXFS.reset() + FakeXFS.attributes_by_inode[os.stat(obj / "staging").st_ino] = (9, True) + self.assertEqual(self.serve(ENSURE), {"error": "foreign_project"}) + FakeXFS.reset() + FakeXFS.usage_by_project[1000] = (4096, 1) + self.assertEqual(self.serve(ENSURE), {"error": "foreign_project"}) + FakeXFS.reset() + self.assertEqual(self.serve(ENSURE)["project_id"], 1000) + FakeXFS.attributes_by_inode[os.stat(obj / "staging").st_ino] = (1000, False) + self.assertEqual(self.serve(ENSURE), {"error": "foreign_project"}) + FakeXFS.attributes_by_inode[os.stat(obj / "staging").st_ino] = (0, False) + self.assertEqual(self.serve(ENSURE), {"error": "foreign_project"}) + + def test_ready_limits_are_verified_not_repaired(self): + self.envelope() + self.serve(ENSURE) + FakeXFS.limits_by_project[1000] = (0, 0, 0, 0, 0, 0) + self.assertEqual(self.serve(ENSURE), {"error": "quota_mismatch"}) + self.assertEqual(FakeXFS.limits_by_project[1000], (0, 0, 0, 0, 0, 0)) + FakeXFS.limits_by_project[1000] = (1 << 20, 2048 << 20, 0, 524288, 0, 0) + self.assertEqual(self.serve(ENSURE), {"error": "quota_mismatch"}) + + def test_different_existing_limits_before_ready_are_refused(self): + self.envelope() + FakeXFS.limits_by_project[1000] = (0, 1 << 30, 0, 524288, 0, 0) + self.assertEqual(self.serve(ENSURE), {"error": "quota_mismatch"}) + + def test_existing_user_data_is_never_projected(self): + obj = self.envelope() + (obj / "staging" / "attempt").mkdir() + self.assertEqual(self.serve(ENSURE), {"error": "nonempty_object"}) + (obj / "staging" / "attempt").rmdir() + (obj / "live").mkdir() + self.assertEqual(self.serve(ENSURE), {"error": "nonempty_object"}) + (obj / "live").rmdir() + (obj / "trash" / "retired").mkdir() + self.assertEqual(self.serve(ENSURE), {"error": "nonempty_object"}) + + def test_deleted_missing_and_unsafe_objects(self): + self.assertEqual(self.serve(ENSURE), {"error": "object_missing"}) + obj = self.envelope() + (obj / "deleted-marker").write_text("deleted\n") + self.assertEqual(self.serve(ENSURE), {"error": "object_deleted"}) + (obj / "deleted-marker").unlink() + (obj / "staging").rmdir() + (obj / "staging").symlink_to(obj / "trash") + self.assertEqual(self.serve(ENSURE), {"error": "unsafe_path"}) + (obj / "staging").unlink() + (obj / "staging").mkdir(mode=0o755) + os.chmod(obj / "staging", 0o755) + self.assertEqual(self.serve(ENSURE), {"error": "unsafe_path"}) + + def test_journal_identity_and_enforcement(self): + self.envelope() + self.assertEqual(self.serve(dict(ENSURE, namespace_id="77777777-7777-4777-8777-777777777777")), {"error": "journal_mismatch"}) + FakeXFS.filesystem = "88888888-8888-4888-8888-888888888888" + self.assertEqual(self.serve(ENSURE), {"error": "journal_mismatch"}) + FakeXFS.filesystem = FILESYSTEM + FakeXFS.enforcement = False + self.assertEqual(self.serve(ENSURE), {"error": "enforcement_off"}) + FakeXFS.enforcement = True + with self.assertRaises(helper.Refused): + helper.init(str(self.config), 1000, 2000) + marker = self.export / ".oac-storage-root" + marker.write_text("99999999-9999-4999-8999-999999999999\n") + self.assertEqual(self.serve(ENSURE), {"error": "journal_mismatch"}) + + def test_invalid_requests(self): + self.envelope() + for request in (dict(ENSURE, object_id="{" + ENSURE["object_id"] + "}"), dict(ENSURE, namespace_id=ENSURE["namespace_id"].replace("2", "A")), dict(ENSURE, extra=1), + dict(ENSURE, capacity_mib=True), dict(ENSURE, capacity_mib=0), dict(ENSURE, operation="delete"), + {"operation": "check"}, dict(ENSURE, tenant_id="00000000-0000-0000-0000-000000000000")): + self.assertEqual(self.serve(request), {"error": "invalid_request"}, request) + self.assertEqual(helper.serve(str(self.config), io.BytesIO(b"{" * 5000)), {"error": "invalid_request"}) + self.assertEqual(helper.serve(str(self.config), io.BytesIO(b"[]")), {"error": "invalid_request"}) + + def test_unsafe_configuration(self): + os.chmod(self.config, 0o666) + self.assertEqual(self.serve(ENSURE), {"error": "unsafe_path"}) + os.chmod(self.config, 0o600) + self.config.write_text(json.dumps({"export_root": str(self.mount), "mount_point": str(self.mount)})) + self.assertEqual(self.serve(ENSURE), {"error": "unsafe_path"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/core/internal/workspacefs/nfs/quota/oac-workspace-quota b/services/core/internal/workspacefs/nfs/quota/oac-workspace-quota new file mode 100755 index 000000000..0b9b8b324 --- /dev/null +++ b/services/core/internal/workspacefs/nfs/quota/oac-workspace-quota @@ -0,0 +1,458 @@ +#!/usr/bin/python3 -I +"""Bind kernel NFS workspace objects to XFS project quotas on the storage host. + +The OpenAgentCore nfs adapter calls this command through a restricted OpenSSH +forced command that runs it as root: + + oac-workspace-quota --config /etc/oac/workspace-quota.json serve + +serve reads one JSON request from stdin and writes one JSON object to stdout. +The request can name no path, project ID, user or command. The operator +initializes the journal once, locally: + + oac-workspace-quota --config /etc/oac/workspace-quota.json init \\ + --projid-start FIRST --projid-end LAST + +Each object's staging directory receives its own project ID and inheritance +before Core creates any stage in it, so live data and retired trash keep that +project; the object's identity, deletion marker and parents stay outside it. +A local SQLite journal on the same XFS filesystem, outside the export, records +the immutable binding of each object to its reference, configuration, capacity, +staging inode and project ID. Request/response fixtures shared with the Go +adapter live in ../testdata/quota. +""" + +import argparse +import errno +import fcntl +import json +import os +import re +import sqlite3 +import stat +import struct +import subprocess +import sys +import uuid + +MIB = 1 << 20 +REQUEST_LIMIT = 4096 +ROOT_MARKER = ".oac-storage-root" +CONTROL_DIRECTORY = ".oac-quota" +PROJINHERIT = 0x200 +FS_IOC_FSGETXATTR = 0x801C581F +FS_IOC_FSSETXATTR = 0x401C5820 +FS_IOC_GETVERSION = 0x80087601 +XFS_IOC_FSGEOMETRY_V1 = 0x80705864 +TOOL_ENVIRONMENT = {"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "LC_ALL": "C"} + +ERROR_CODES = frozenset({ + "invalid_request", "object_deleted", "identity_mismatch", "binding_mismatch", "object_replaced", + "foreign_project", "nonempty_object", "quota_mismatch", "unsafe_path", "object_missing", + "enforcement_off", "range_exhausted", "journal_mismatch", "operation_failed", +}) +ENSURE_FIELDS = frozenset({"operation", "namespace_id", "tenant_id", "environment_id", "object_id", + "configuration_id", "capacity_mib", "inodes_hard"}) + + +class Refused(Exception): + def __init__(self, code): + assert code in ERROR_CODES, code + super().__init__(code) + self.code = code + + +def require(condition, code): + if not condition: + raise Refused(code) + + +def step(name): + """Crash-injection point for tests; production does nothing.""" + + +def canonical_uuid(value): + if type(value) is not str: + return False + try: + parsed = uuid.UUID(value) + except ValueError: + return False + return parsed.int != 0 and str(parsed) == value + + +def bounded_int(value, low, high): + return type(value) is int and low <= value <= high + + +class XFS: + """Privileged XFS operations. Tests substitute an equivalent fake.""" + + def __init__(self, mount_point=None, lock_fd=None): + self.mount_point, self.lock_fd = mount_point, lock_fd + + def filesystem_uuid(self, fd): + geometry = fcntl.ioctl(fd, XFS_IOC_FSGEOMETRY_V1, bytes(112)) + return str(uuid.UUID(bytes=geometry[64:80])) + + def attributes(self, fd): + flags, _, _, project, _ = struct.unpack("=5I8x", fcntl.ioctl(fd, FS_IOC_FSGETXATTR, bytes(28))) + return project, bool(flags & PROJINHERIT) + + def project(self, fd, project): + # One FS_IOC_FSSETXATTR sets the project and inheritance together. + fields = list(struct.unpack("=5I8x", fcntl.ioctl(fd, FS_IOC_FSGETXATTR, bytes(28)))) + fields[0] |= PROJINHERIT + fields[3] = project + fcntl.ioctl(fd, FS_IOC_FSSETXATTR, struct.pack("=5I8x", *fields)) + + def generation(self, fd): + return struct.unpack("=q", fcntl.ioctl(fd, FS_IOC_GETVERSION, bytes(8)))[0] + + def _quota(self, command): + # The child inherits the helper lock, so it remains held if the helper dies. + result = subprocess.run(["xfs_quota", "-x", "-c", command, self.mount_point], capture_output=True, text=True, + timeout=20, env=TOOL_ENVIRONMENT, pass_fds=(self.lock_fd,), stdin=subprocess.DEVNULL) + require(result.returncode == 0 and not result.stderr.strip(), "operation_failed") + return result.stdout + + def enforced(self): + output = self._quota("state -p") + return (len(re.findall(r"^\s*Accounting: ON$", output, re.M)) == 1 + and len(re.findall(r"^\s*Enforcement: ON$", output, re.M)) == 1) + + def limits(self, project): + """Return (bsoft, bhard, isoft, ihard, rtbsoft, rtbhard) in bytes and inodes. + A project without a dquot reports zero limits; that never means the ID is unused.""" + rows = [line.split() for line in self._quota("dump -p -L %d -U %d" % (project, project)).splitlines() + if line.strip() and not line.startswith("fs = ")] + if not rows: + return (0, 0, 0, 0, 0, 0) + require(len(rows) == 1 and len(rows[0]) in (5, 7) and all(v.isdigit() for v in rows[0]), "operation_failed") + values = [int(v) for v in rows[0]] + [0, 0] + require(values[0] == project, "operation_failed") + bsoft, bhard, isoft, ihard, rtsoft, rthard = values[1:7] + return (bsoft * 512, bhard * 512, isoft, ihard, rtsoft * 512, rthard * 512) + + def usage(self, project): + """Return (bytes, inodes) charged to project; zero without a dquot.""" + lines = [line for line in self._quota("quota -p -N -b -i %d" % project).splitlines() if line.strip()] + if not lines: + return (0, 0) + match = re.fullmatch(r"\S+\s+(\d+)\s+\d+\s+\d+\s+\d+\s+\[[^\]]*\]\s+(\d+)\s+\d+\s+\d+\s+\d+\s+\[[^\]]*\]\s+\S+", + lines[0].strip()) if len(lines) == 1 else None + require(match is not None, "operation_failed") + return (int(match.group(1)) * 1024, int(match.group(2))) + + def set_limits(self, project, bytes_hard, inodes_hard): + self._quota("limit -p bsoft=0 bhard=%dm isoft=0 ihard=%d %d" % (bytes_hard // MIB, inodes_hard, project)) + + +BACKEND = XFS + + +def load_config(path): + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + info = os.fstat(fd) + require(stat.S_ISREG(info.st_mode) and info.st_uid == os.geteuid() and info.st_mode & 0o022 == 0 + and info.st_size <= REQUEST_LIMIT, "unsafe_path") + config = json.loads(os.read(fd, REQUEST_LIMIT + 1)) + finally: + os.close(fd) + require(type(config) is dict and set(config) == {"export_root", "mount_point"}, "unsafe_path") + export, mount = config["export_root"], config["mount_point"] + for value in (export, mount): + require(type(value) is str and os.path.isabs(value) and os.path.normpath(value) == value and "\0" not in value, + "unsafe_path") + control = os.path.join(mount, CONTROL_DIRECTORY) + require(export != mount and os.path.commonpath([export, mount]) == mount + and os.path.commonpath([export, control]) != control, "unsafe_path") + return export, mount, control + + +def open_directory(name, dir_fd=None): + return os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=dir_fd) + + +def private_directory(fd, uid, device): + info = os.fstat(fd) + require(stat.S_ISDIR(info.st_mode) and info.st_uid == uid and stat.S_IMODE(info.st_mode) == 0o700 + and info.st_dev == device, "unsafe_path") + return info + + +def read_private(name, dir_fd, uid, limit): + try: + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=dir_fd) + except FileNotFoundError: + return None + try: + info = os.fstat(fd) + require(stat.S_ISREG(info.st_mode) and info.st_uid == uid and stat.S_IMODE(info.st_mode) == 0o600 + and info.st_size <= limit, "unsafe_path") + return os.read(fd, limit + 1) + finally: + os.close(fd) + + +def exists(name, dir_fd): + try: + os.stat(name, dir_fd=dir_fd, follow_symlinks=False) + return True + except FileNotFoundError: + return False + + +class Storage: + """The pinned export, its filesystem and the journal, under the helper lock.""" + + def __init__(self, config_path, create_control=False): + self.export, self.mount, self.control = load_config(config_path) + self.descriptors = [] + try: + self._open(create_control) + except BaseException: + self.close() + raise + + def directory(self, name, dir_fd=None): + fd = open_directory(name, dir_fd) + self.descriptors.append(fd) + return fd + + def close(self): + for resource in reversed(self.descriptors): + resource.close() if isinstance(resource, sqlite3.Connection) else os.close(resource) + self.descriptors = [] + + def _open(self, create_control): + self.mount_fd = self.directory(self.mount) + self.device = os.fstat(self.mount_fd).st_dev + if create_control: + try: + os.mkdir(CONTROL_DIRECTORY, 0o700, dir_fd=self.mount_fd) + except FileExistsError: + pass + try: + self.control_fd = self.directory(CONTROL_DIRECTORY, self.mount_fd) + except FileNotFoundError: + raise Refused("journal_mismatch") + private_directory(self.control_fd, os.geteuid(), self.device) + self.lock_fd = os.open("lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=self.control_fd) + self.descriptors.append(self.lock_fd) + fcntl.flock(self.lock_fd, fcntl.LOCK_EX) + self.xfs = BACKEND(self.mount, self.lock_fd) + self.root_fd = self.directory(self.export) + info = os.fstat(self.root_fd) + require(stat.S_ISDIR(info.st_mode) and info.st_dev == self.device and info.st_uid != 0, "unsafe_path") + self.uid = info.st_uid + private_directory(self.root_fd, self.uid, self.device) + self.filesystem = self.xfs.filesystem_uuid(self.mount_fd) + + def namespace(self): + raw = read_private(ROOT_MARKER, self.root_fd, self.uid, 64) + require(raw is not None and raw.endswith(b"\n"), "journal_mismatch") + value = raw[:-1].decode("ascii", "replace") + require(canonical_uuid(value), "journal_mismatch") + return value + + def journal(self, create=False): + # The control directory is private to root and was verified above. + if not create: + require(exists("journal.sqlite", self.control_fd), "journal_mismatch") + db = sqlite3.connect(os.path.join(self.control, "journal.sqlite"), isolation_level=None, timeout=20) + self.descriptors.insert(0, db) + db.execute("PRAGMA journal_mode=DELETE") + db.execute("PRAGMA synchronous=FULL") + return db + + +SCHEMA = """ +CREATE TABLE meta ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + filesystem_uuid TEXT NOT NULL, namespace_id TEXT NOT NULL, + projid_start INTEGER NOT NULL, projid_end INTEGER NOT NULL, next_projid INTEGER NOT NULL, + CHECK (0 < projid_start AND projid_start <= projid_end AND projid_end < 4294967295), + CHECK (projid_start <= next_projid AND next_projid <= projid_end + 1) +); +CREATE TABLE objects ( + object_id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL, environment_id TEXT NOT NULL, + configuration_id TEXT NOT NULL, capacity_mib INTEGER NOT NULL, inodes_hard INTEGER NOT NULL, + staging_inode INTEGER NOT NULL, staging_generation INTEGER NOT NULL, + projid INTEGER NOT NULL UNIQUE, + state TEXT NOT NULL CHECK (state IN ('allocated', 'projected', 'ready')) +); +""" + + +def meta(storage, db, namespace): + row = db.execute("SELECT filesystem_uuid, namespace_id, projid_end, next_projid FROM meta").fetchone() + require(row is not None and row[0] == storage.filesystem and row[1] == namespace + and storage.namespace() == namespace, "journal_mismatch") + require(storage.xfs.enforced(), "enforcement_off") + return row + + +def check(storage, request): + require(set(request) == {"operation", "namespace_id"} and canonical_uuid(request["namespace_id"]), "invalid_request") + db = storage.journal() + _, _, end, following = meta(storage, db, request["namespace_id"]) + require(following <= end, "range_exhausted") + return {"filesystem_uuid": storage.filesystem} + + +def ensure(storage, request): + require(set(request) == ENSURE_FIELDS and all(canonical_uuid(request[k]) for k in + ("namespace_id", "tenant_id", "environment_id", "object_id", "configuration_id")) + and bounded_int(request["capacity_mib"], 1, 4294967295) + and bounded_int(request["inodes_hard"], 1, (1 << 63) - 1), "invalid_request") + db = storage.journal() + meta(storage, db, request["namespace_id"]) + reference = {k: request[k] for k in ("tenant_id", "environment_id", "object_id")} + capacity, inodes = request["capacity_mib"], request["inodes_hard"] + try: + objects_fd = storage.directory("objects", storage.root_fd) + object_fd = storage.directory(request["object_id"], objects_fd) + staging_fd = storage.directory("staging", object_fd) + except FileNotFoundError: + raise Refused("object_missing") + except OSError as error: + if error.errno in (errno.ELOOP, errno.ENOTDIR): + raise Refused("unsafe_path") + raise + for fd in (objects_fd, object_fd, staging_fd): + private_directory(fd, storage.uid, storage.device) + raw = read_private("identity", object_fd, storage.uid, REQUEST_LIMIT) + require(raw is not None, "object_missing") + try: + identity = json.loads(raw) + except ValueError: + raise Refused("identity_mismatch") + require(identity == reference, "identity_mismatch") + require(not exists("deleted-marker", object_fd), "object_deleted") + for fd in (objects_fd, object_fd): + require(storage.xfs.attributes(fd) == (0, False), "foreign_project") + staging = os.fstat(staging_fd) + inode, generation = staging.st_ino, storage.xfs.generation(staging_fd) + + # The helper lock serializes every journal writer. + row = db.execute("SELECT tenant_id, environment_id, configuration_id, capacity_mib, inodes_hard, staging_inode," + " staging_generation, projid, state FROM objects WHERE object_id = ?", (request["object_id"],)).fetchone() + if row is None: + # Never project existing user data: only a fresh empty envelope is bound. + require(not os.listdir(staging_fd) and not exists("live", object_fd), "nonempty_object") + if exists("trash", object_fd): + trash_fd = storage.directory("trash", object_fd) + require(not os.listdir(trash_fd), "nonempty_object") + require(storage.xfs.attributes(staging_fd) == (0, False), "foreign_project") + _, _, end, project = db.execute("SELECT filesystem_uuid, namespace_id, projid_end, next_projid FROM meta").fetchone() + require(project <= end, "range_exhausted") + # The allocation commits before any XFS change and is never reused. + with db: + db.execute("BEGIN IMMEDIATE") + db.execute("UPDATE meta SET next_projid = next_projid + 1") + db.execute("INSERT INTO objects VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'allocated')", + (request["object_id"], request["tenant_id"], request["environment_id"], request["configuration_id"], + capacity, inodes, inode, generation, project)) + row = (request["tenant_id"], request["environment_id"], request["configuration_id"], capacity, inodes, inode, + generation, project, "allocated") + step("allocated") + require(row[:5] == (request["tenant_id"], request["environment_id"], request["configuration_id"], capacity, inodes), + "binding_mismatch") + require(row[5:7] == (inode, generation), "object_replaced") + project, state = row[7], row[8] + + current = storage.xfs.attributes(staging_fd) + if state == "allocated" and current == (0, False): + # A charged ID that this journal never projected belongs to someone else. + require(storage.xfs.usage(project) == (0, 0), "foreign_project") + require(not os.listdir(staging_fd), "nonempty_object") + storage.xfs.project(staging_fd, project) + step("projected-inode") + current = storage.xfs.attributes(staging_fd) + require(current == (project, True), "foreign_project") + if state == "allocated": + db.execute("UPDATE objects SET state = 'projected' WHERE object_id = ?", (request["object_id"],)) + state = "projected" + step("projected") + + wanted = (0, capacity * MIB, 0, inodes, 0, 0) + limits = storage.xfs.limits(project) + if state == "projected" and limits == (0, 0, 0, 0, 0, 0): + storage.xfs.set_limits(project, capacity * MIB, inodes) + step("limited") + limits = storage.xfs.limits(project) + # A ready object whose limits changed, including lost limits, is refused + # rather than repaired; the supported restore preserves them. + require(limits == wanted, "quota_mismatch") + if state == "projected": + db.execute("UPDATE objects SET state = 'ready' WHERE object_id = ?", (request["object_id"],)) + step("ready") + require(not exists("deleted-marker", object_fd), "object_deleted") + return {"project_id": project, "bytes_hard": capacity * MIB, "inodes_hard": inodes, "filesystem_uuid": storage.filesystem} + + +def serve(config_path, stdin): + raw = stdin.read(REQUEST_LIMIT + 1) + try: + require(len(raw) <= REQUEST_LIMIT, "invalid_request") + try: + request = json.loads(raw) + except ValueError: + raise Refused("invalid_request") + require(type(request) is dict and request.get("operation") in ("check", "ensure"), "invalid_request") + storage = Storage(config_path) + try: + return (check if request["operation"] == "check" else ensure)(storage, request) + finally: + storage.close() + except Refused as refused: + return {"error": refused.code} + except (OSError, sqlite3.Error, subprocess.SubprocessError, UnicodeError): + return {"error": "operation_failed"} + + +def init(config_path, start, end): + require(bounded_int(start, 1, 4294967294) and bounded_int(end, start, 4294967294), "invalid_request") + storage = Storage(config_path, create_control=True) + try: + require(not exists("journal.sqlite", storage.control_fd), "journal_mismatch") + namespace = storage.namespace() + require(storage.xfs.enforced(), "enforcement_off") + db = storage.journal(create=True) + with db: + db.execute("BEGIN IMMEDIATE") + for statement in SCHEMA.split(";"): + if statement.strip(): + db.execute(statement) + db.execute("INSERT INTO meta VALUES (1, ?, ?, ?, ?, ?)", (storage.filesystem, namespace, start, end, start)) + os.fsync(storage.control_fd) + finally: + storage.close() + return {"filesystem_uuid": storage.filesystem, "namespace_id": namespace, "projid_start": start, "projid_end": end} + + +def main(argv=None): + parser = argparse.ArgumentParser(prog="oac-workspace-quota") + parser.add_argument("--config", default="/etc/oac/workspace-quota.json") + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("serve") + initialize = commands.add_parser("init") + initialize.add_argument("--projid-start", type=int, required=True) + initialize.add_argument("--projid-end", type=int, required=True) + arguments = parser.parse_args(argv) + if arguments.command == "serve": + result = serve(arguments.config, sys.stdin.buffer) + else: + try: + result = init(arguments.config, arguments.projid_start, arguments.projid_end) + except Refused as refused: + result = {"error": refused.code} + sys.stdout.write(json.dumps(result, sort_keys=True) + "\n") + sys.stdout.flush() + return 1 if "error" in result else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/services/core/internal/workspacefs/nfs/quota_test.go b/services/core/internal/workspacefs/nfs/quota_test.go new file mode 100644 index 000000000..7ef17573f --- /dev/null +++ b/services/core/internal/workspacefs/nfs/quota_test.go @@ -0,0 +1,313 @@ +package nfs + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/json" + "encoding/pem" + "errors" + "io" + "net" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" + "golang.org/x/crypto/ssh" +) + +func fixture(t *testing.T, name string) []byte { + t.Helper() + raw, err := os.ReadFile("testdata/quota/" + name) + if err != nil { + t.Fatal(err) + } + return bytes.TrimSpace(raw) +} + +func newKey(t *testing.T) (ssh.Signer, string) { + t.Helper() + _, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + signer, err := ssh.NewSignerFromKey(private) + if err != nil { + t.Fatal(err) + } + block, err := ssh.MarshalPrivateKey(private, "") + if err != nil { + t.Fatal(err) + } + return signer, string(pem.EncodeToMemory(block)) +} + +// fakeHelper is an SSH server standing in for the forced helper command. +type fakeHelper struct { + t *testing.T + address string + hostKey string + mu sync.Mutex + requests [][]byte + respond func([]byte) []byte +} + +func startHelper(t *testing.T, client ssh.PublicKey, respond func([]byte) []byte) *fakeHelper { + t.Helper() + host, _ := newKey(t) + config := &ssh.ServerConfig{PublicKeyCallback: func(_ ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) { + if !bytes.Equal(key.Marshal(), client.Marshal()) { + return nil, errors.New("unauthorized") + } + return nil, nil + }} + config.AddHostKey(host) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { listener.Close() }) + f := &fakeHelper{t: t, address: listener.Addr().String(), hostKey: strings.TrimSpace(string(ssh.MarshalAuthorizedKey(host.PublicKey()))), respond: respond} + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go f.serve(conn, config) + } + }() + return f +} + +func (f *fakeHelper) serve(conn net.Conn, config *ssh.ServerConfig) { + defer conn.Close() + _, channels, requests, err := ssh.NewServerConn(conn, config) + if err != nil { + return + } + go ssh.DiscardRequests(requests) + for candidate := range channels { + channel, requests, err := candidate.Accept() + if err != nil { + return + } + go func() { + for request := range requests { + _ = request.Reply(request.Type == "exec", nil) + if request.Type != "exec" { + continue + } + input, _ := io.ReadAll(channel) + f.mu.Lock() + f.requests = append(f.requests, input) + f.mu.Unlock() + output := f.respond(input) + if output == nil { + time.Sleep(time.Minute) + } + _, _ = channel.Write(output) + _, _ = channel.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{0})) + channel.Close() + } + }() + } +} + +func (f *fakeHelper) received() [][]byte { + f.mu.Lock() + defer f.mu.Unlock() + return append([][]byte(nil), f.requests...) +} + +func quotaConfiguration(t *testing.T, address, hostKey string) json.RawMessage { + t.Helper() + raw, err := json.Marshal(map[string]any{"root": "/nfs/workspaces", "namespace_id": "22222222-2222-4222-8222-222222222222", "uid": 65532, + "quota": map[string]string{"endpoint": address, "user": "oac-quota", "host_key": hostKey}}) + if err != nil { + t.Fatal(err) + } + return raw +} + +func credentialInput(key string) json.RawMessage { + raw, _ := json.Marshal(map[string]string{"ssh_private_key": key}) + return raw +} + +func quotaControl(t *testing.T, parameters json.RawMessage, credential json.RawMessage) *Filesystem { + t.Helper() + canonical, encoded, err := Registration{}.Normalize(parameters, credential) + if err != nil { + t.Fatal(err) + } + control, err := Registration{}.Control(workspacefs.ConfigurationRecord{Configuration: workspacefs.Configuration{ID: "11111111-1111-4111-8111-111111111111", Adapter: Name, Parameters: canonical}, Credential: encoded}) + if err != nil { + t.Fatal(err) + } + return control.(*Filesystem) +} + +var fixtureReference = workspacefs.Reference{TenantID: "33333333-3333-4333-8333-333333333333", EnvironmentID: "44444444-4444-4444-8444-444444444444", ObjectID: "55555555-5555-4555-8555-555555555555"} + +func TestQuotaConfigurationCodec(t *testing.T) { + _, key := newKey(t) + host, _ := newKey(t) + hostKey := strings.TrimSpace(string(ssh.MarshalAuthorizedKey(host.PublicKey()))) + plain := json.RawMessage(`{"root":"/nfs/workspaces","namespace_id":"22222222-2222-4222-8222-222222222222","uid":65532}`) + quota := quotaConfiguration(t, "storage.internal:22", hostKey+" root@storage") + for name, input := range map[string][2]json.RawMessage{ + "quota without credential": {quota, nil}, + "credential without quota": {plain, credentialInput(key)}, + "null credential": {quota, json.RawMessage(`null`)}, + "unknown credential field": {quota, json.RawMessage(`{"ssh_private_key":"x","password":"y"}`)}, + "invalid private key": {quota, credentialInput("not a key")}, + "host key options": {quotaConfiguration(t, "storage.internal:22", `restrict `+hostKey), credentialInput(key)}, + "endpoint without port": {quotaConfiguration(t, "storage.internal", hostKey), credentialInput(key)}, + "zero port": {quotaConfiguration(t, "storage.internal:0", hostKey), credentialInput(key)}, + "nested unknown parameter": {json.RawMessage(strings.Replace(string(quota), `"user"`, `"identity_file":"/key","user"`, 1)), credentialInput(key)}, + "nested null parameter": {json.RawMessage(strings.Replace(string(quota), `"user":"oac-quota"`, `"user":null`, 1)), credentialInput(key)}, + "explicit null quota": {json.RawMessage(strings.Replace(string(plain), `}`, `,"quota":null}`, 1)), nil}, + "user with shell character": {json.RawMessage(strings.Replace(string(quota), `oac-quota`, `oac;quota`, 1)), credentialInput(key)}, + } { + if _, _, err := (Registration{}).Normalize(input[0], input[1]); !errors.Is(err, workspacefs.ErrInvalid) { + t.Fatalf("%s accepted: %v", name, err) + } + } + canonical, encoded, err := Registration{}.Normalize(quota, credentialInput(key)) + if err != nil || strings.Contains(string(canonical), "root@storage") || strings.Contains(string(canonical), "PRIVATE") { + t.Fatalf("canonical parameters %s %v", canonical, err) + } + again, reencoded, err := Registration{}.Normalize(canonical, encoded) + if err != nil || !bytes.Equal(again, canonical) || !bytes.Equal(reencoded, encoded) { + t.Fatal("encoded credential does not encode to itself", err) + } + if _, _, err := (Registration{}).Normalize(plain, nil); err != nil { + t.Fatal("existing configuration without credential rejected", err) + } + config := workspacefs.Configuration{ID: "11111111-1111-4111-8111-111111111111", Adapter: Name, Parameters: canonical} + if _, err := (Registration{}).Control(workspacefs.ConfigurationRecord{Configuration: config}); !errors.Is(err, workspacefs.ErrInvalid) { + t.Fatal("quota Control constructed without credential", err) + } + control, err := Registration{}.Control(workspacefs.ConfigurationRecord{Configuration: config, Credential: encoded}) + if err != nil || !control.Declaration().CapacityQuota { + t.Fatal("quota declaration", err) + } + resolver, err := Registration{}.Resolver(config) + if err != nil || resolver.(*Filesystem).quota != nil { + t.Fatal("resolver requires or holds a credential", err) + } + plainControl, err := Registration{}.Control(workspacefs.ConfigurationRecord{Configuration: workspacefs.Configuration{ID: config.ID, Adapter: Name, Parameters: plain}}) + if err != nil || plainControl.Declaration().CapacityQuota { + t.Fatal("plain declaration", err) + } + if _, err := (Registration{}).Control(workspacefs.ConfigurationRecord{Configuration: workspacefs.Configuration{ID: config.ID, Adapter: Name, Parameters: plain}, Credential: encoded}); !errors.Is(err, workspacefs.ErrInvalid) { + t.Fatal("plain Control accepted a credential", err) + } +} + +func TestQuotaWireMatchesSharedFixtures(t *testing.T) { + signer, key := newKey(t) + helper := startHelper(t, signer.PublicKey(), func(request []byte) []byte { + if bytes.Contains(request, []byte(`"check"`)) { + return append(fixture(t, "check-response.json"), '\n') + } + return append(fixture(t, "ensure-response.json"), '\n') + }) + f := quotaControl(t, quotaConfiguration(t, helper.address, helper.hostKey), credentialInput(key)) + if err := f.quota.check(t.Context(), f.parameters.NamespaceID); err != nil { + t.Fatal(err) + } + if err := f.quota.ensure(t.Context(), fixtureReference, f.configuration.ID, f.parameters.NamespaceID, 2048); err != nil { + t.Fatal(err) + } + got := helper.received() + if len(got) != 2 || !bytes.Equal(got[0], fixture(t, "check-request.json")) || !bytes.Equal(got[1], fixture(t, "ensure-request.json")) { + t.Fatalf("requests differ from shared fixtures: %q", got) + } + // A receipt for a different bound is never accepted as enforcement. + if err := f.quota.ensure(t.Context(), fixtureReference, f.configuration.ID, f.parameters.NamespaceID, 4096); !errors.Is(err, workspacefs.ErrOwnership) { + t.Fatal(err) + } +} + +func TestQuotaErrorsMatchSharedFixture(t *testing.T) { + var codes map[string]string + if err := json.Unmarshal(fixture(t, "errors.json"), &codes); err != nil { + t.Fatal(err) + } + classes := map[string]error{"invalid": workspacefs.ErrInvalid, "not_found": workspacefs.ErrNotFound, "ownership": workspacefs.ErrOwnership, "unavailable": workspacefs.ErrUnavailable} + if len(codes) != len(quotaErrors) { + t.Fatalf("helper error codes %d, adapter mapping %d", len(codes), len(quotaErrors)) + } + for code, class := range codes { + if quotaErrors[code] != classes[class] || classes[class] == nil { + t.Fatalf("%s maps to %v, fixture says %s", code, quotaErrors[code], class) + } + } + signer, key := newKey(t) + response := `{"error": "object_deleted"}` + helper := startHelper(t, signer.PublicKey(), func([]byte) []byte { return []byte(response + "\n") }) + f := quotaControl(t, quotaConfiguration(t, helper.address, helper.hostKey), credentialInput(key)) + if err := f.quota.ensure(t.Context(), fixtureReference, f.configuration.ID, f.parameters.NamespaceID, 2048); !errors.Is(err, workspacefs.ErrNotFound) { + t.Fatal(err) + } + response = `{"error": "not_a_code"}` + if err := f.quota.ensure(t.Context(), fixtureReference, f.configuration.ID, f.parameters.NamespaceID, 2048); !errors.Is(err, workspacefs.ErrUnconfirmed) { + t.Fatal(err) + } +} + +func TestQuotaHostKeyIsPinned(t *testing.T) { + signer, key := newKey(t) + helper := startHelper(t, signer.PublicKey(), func([]byte) []byte { return []byte("{}\n") }) + impostor, _ := newKey(t) + f := quotaControl(t, quotaConfiguration(t, helper.address, strings.TrimSpace(string(ssh.MarshalAuthorizedKey(impostor.PublicKey())))), credentialInput(key)) + if err := f.quota.ensure(t.Context(), fixtureReference, f.configuration.ID, f.parameters.NamespaceID, 2048); !errors.Is(err, workspacefs.ErrUnavailable) || errors.Is(err, workspacefs.ErrUnconfirmed) { + t.Fatal("unpinned host accepted", err) + } + if len(helper.received()) != 0 { + t.Fatal("request sent to an unverified host") + } + _, other := newKey(t) + f = quotaControl(t, quotaConfiguration(t, helper.address, helper.hostKey), credentialInput(other)) + if err := f.quota.check(t.Context(), f.parameters.NamespaceID); !errors.Is(err, workspacefs.ErrUnavailable) { + t.Fatal("unauthorized client key accepted", err) + } +} + +func TestQuotaUnknownOutcomes(t *testing.T) { + signer, key := newKey(t) + var output []byte + helper := startHelper(t, signer.PublicKey(), func([]byte) []byte { return output }) + f := quotaControl(t, quotaConfiguration(t, helper.address, helper.hostKey), credentialInput(key)) + for name, value := range map[string][]byte{"malformed": []byte("Traceback\n"), "empty": {}, "unknown field": []byte(`{"filesystem_uuid":"66666666-6666-4666-8666-666666666666","extra":1}`)} { + output = value + if err := f.quota.ensure(t.Context(), fixtureReference, f.configuration.ID, f.parameters.NamespaceID, 2048); !errors.Is(err, workspacefs.ErrUnconfirmed) { + t.Fatal(name, "ensure", err) + } + if err := f.quota.check(t.Context(), f.parameters.NamespaceID); !errors.Is(err, workspacefs.ErrUnavailable) || errors.Is(err, workspacefs.ErrUnconfirmed) { + t.Fatal(name, "check", err) + } + } + output = nil + ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) + defer cancel() + if err := f.quota.ensure(ctx, fixtureReference, f.configuration.ID, f.parameters.NamespaceID, 2048); !errors.Is(err, workspacefs.ErrUnconfirmed) { + t.Fatal("timed out mutation reported settled", err) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + closed := listener.Addr().String() + listener.Close() + f = quotaControl(t, quotaConfiguration(t, closed, helper.hostKey), credentialInput(key)) + if err := f.quota.ensure(t.Context(), fixtureReference, f.configuration.ID, f.parameters.NamespaceID, 2048); !errors.Is(err, workspacefs.ErrUnavailable) || errors.Is(err, workspacefs.ErrUnconfirmed) { + t.Fatal("unreachable server reported unconfirmed", err) + } +} diff --git a/services/core/internal/workspacefs/nfs/testdata/quota/check-request.json b/services/core/internal/workspacefs/nfs/testdata/quota/check-request.json new file mode 100644 index 000000000..8ab03c960 --- /dev/null +++ b/services/core/internal/workspacefs/nfs/testdata/quota/check-request.json @@ -0,0 +1 @@ +{"operation":"check","namespace_id":"22222222-2222-4222-8222-222222222222"} diff --git a/services/core/internal/workspacefs/nfs/testdata/quota/check-response.json b/services/core/internal/workspacefs/nfs/testdata/quota/check-response.json new file mode 100644 index 000000000..dcac818ad --- /dev/null +++ b/services/core/internal/workspacefs/nfs/testdata/quota/check-response.json @@ -0,0 +1 @@ +{"filesystem_uuid": "66666666-6666-4666-8666-666666666666"} diff --git a/services/core/internal/workspacefs/nfs/testdata/quota/ensure-request.json b/services/core/internal/workspacefs/nfs/testdata/quota/ensure-request.json new file mode 100644 index 000000000..51851dfc3 --- /dev/null +++ b/services/core/internal/workspacefs/nfs/testdata/quota/ensure-request.json @@ -0,0 +1 @@ +{"operation":"ensure","namespace_id":"22222222-2222-4222-8222-222222222222","tenant_id":"33333333-3333-4333-8333-333333333333","environment_id":"44444444-4444-4444-8444-444444444444","object_id":"55555555-5555-4555-8555-555555555555","configuration_id":"11111111-1111-4111-8111-111111111111","capacity_mib":2048,"inodes_hard":524288} diff --git a/services/core/internal/workspacefs/nfs/testdata/quota/ensure-response.json b/services/core/internal/workspacefs/nfs/testdata/quota/ensure-response.json new file mode 100644 index 000000000..99bbf7136 --- /dev/null +++ b/services/core/internal/workspacefs/nfs/testdata/quota/ensure-response.json @@ -0,0 +1 @@ +{"bytes_hard": 2147483648, "filesystem_uuid": "66666666-6666-4666-8666-666666666666", "inodes_hard": 524288, "project_id": 1000} diff --git a/services/core/internal/workspacefs/nfs/testdata/quota/errors.json b/services/core/internal/workspacefs/nfs/testdata/quota/errors.json new file mode 100644 index 000000000..585611a96 --- /dev/null +++ b/services/core/internal/workspacefs/nfs/testdata/quota/errors.json @@ -0,0 +1,16 @@ +{ + "binding_mismatch": "ownership", + "enforcement_off": "unavailable", + "foreign_project": "ownership", + "identity_mismatch": "ownership", + "invalid_request": "invalid", + "journal_mismatch": "unavailable", + "nonempty_object": "ownership", + "object_deleted": "not_found", + "object_missing": "unavailable", + "object_replaced": "ownership", + "operation_failed": "unavailable", + "quota_mismatch": "ownership", + "range_exhausted": "unavailable", + "unsafe_path": "ownership" +} diff --git a/services/core/internal/workspacefs/providers/registry.go b/services/core/internal/workspacefs/providers/registry.go index 7d19c785a..593a31bea 100644 --- a/services/core/internal/workspacefs/providers/registry.go +++ b/services/core/internal/workspacefs/providers/registry.go @@ -10,58 +10,46 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs/nfs" ) -type adapter interface { - workspacefs.Control - workspacefs.Resolver +// Registry uses the same registrations in Core and on nodes. Each registration +// implements every workspacefs.Adapter role; Core constructs Control from a +// credentialed record and nodes construct Resolver from public configuration. +type Registry struct { + entries map[string]workspacefs.Adapter } -type registration struct { - normalize func(json.RawMessage) (json.RawMessage, error) - open func(workspacefs.Configuration) (adapter, error) -} - -// Registry uses the same registrations for control and node resolution. A -// registration supplies both protocol roles, never an optional side interface. -type Registry struct{ entries map[string]registration } - func New() *Registry { - return &Registry{entries: map[string]registration{ - nfs.Name: { - normalize: nfs.CanonicalParameters, - open: func(c workspacefs.Configuration) (adapter, error) { - return nfs.New(c) - }, - }, - }} + return &Registry{entries: map[string]workspacefs.Adapter{nfs.Name: nfs.Registration{}}} } -func (r *Registry) lookup(name string) (registration, error) { +func (r *Registry) lookup(name string) (workspacefs.Adapter, error) { if r != nil { if value, ok := r.entries[name]; ok { return value, nil } } - return registration{}, fmt.Errorf("%w: filesystem adapter is not registered", workspacefs.ErrUnsupported) + return nil, fmt.Errorf("%w: filesystem adapter is not registered", workspacefs.ErrUnsupported) } -func (r *Registry) Normalize(c workspacefs.Configuration) (workspacefs.Configuration, error) { +// Normalize canonicalizes a candidate configuration and encodes its credential input. +func (r *Registry) Normalize(c workspacefs.Configuration, credential json.RawMessage) (workspacefs.ConfigurationRecord, error) { if err := c.Validate(); err != nil { - return workspacefs.Configuration{}, err + return workspacefs.ConfigurationRecord{}, err } a, err := r.lookup(c.Adapter) if err != nil { - return workspacefs.Configuration{}, err + return workspacefs.ConfigurationRecord{}, err } - c.Parameters, err = a.normalize(c.Parameters) + record := workspacefs.ConfigurationRecord{Configuration: c} + record.Parameters, record.Credential, err = a.Normalize(c.Parameters, credential) if err != nil { - return workspacefs.Configuration{}, err + return workspacefs.ConfigurationRecord{}, err } - return c, nil + return record, nil } -// Control constructs an adapter from its canonical retained configuration. +// Control constructs an adapter from its canonical retained record. // Construction performs no filesystem operation; Check owns live qualification. -func (r *Registry) Control(c workspacefs.Configuration) (workspacefs.Control, error) { +func (r *Registry) Control(c workspacefs.ConfigurationRecord) (workspacefs.Control, error) { if err := c.Validate(); err != nil { return nil, err } @@ -69,9 +57,10 @@ func (r *Registry) Control(c workspacefs.Configuration) (workspacefs.Control, er if err != nil { return nil, err } - return a.open(c) + return a.Control(c) } +// Resolve validates a transient binding with the public configuration only. func (r *Registry) Resolve(ctx context.Context, binding workspacefs.Binding) (workspacefs.Directory, error) { if err := binding.Validate(); err != nil { return workspacefs.Directory{}, err @@ -80,9 +69,9 @@ func (r *Registry) Resolve(ctx context.Context, binding workspacefs.Binding) (wo if err != nil { return workspacefs.Directory{}, err } - instance, err := a.open(binding.Configuration) + resolver, err := a.Resolver(binding.Configuration) if err != nil { return workspacefs.Directory{}, err } - return instance.Resolve(ctx, binding) + return resolver.Resolve(ctx, binding) } diff --git a/services/core/internal/workspacefs/providers/registry_test.go b/services/core/internal/workspacefs/providers/registry_test.go new file mode 100644 index 000000000..3922bddd0 --- /dev/null +++ b/services/core/internal/workspacefs/providers/registry_test.go @@ -0,0 +1,34 @@ +package providers + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" +) + +func TestRegistryRoles(t *testing.T) { + r := New() + quota := json.RawMessage(`{"root":"/nfs/workspaces","namespace_id":"22222222-2222-4222-8222-222222222222","uid":65532,"quota":{"endpoint":"storage.internal:22","user":"oac-quota","host_key":"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKEJUu1hJdEp2CdwRXbFxsHlMQpWSKMKjvP3w1Frd3gk"}}`) + config := workspacefs.Configuration{ID: "11111111-1111-4111-8111-111111111111", Adapter: "nfs", Parameters: quota} + if _, err := r.Normalize(config, nil); !errors.Is(err, workspacefs.ErrInvalid) { + t.Fatal("quota configuration normalized without its credential", err) + } + if _, err := r.Control(workspacefs.ConfigurationRecord{Configuration: config}); !errors.Is(err, workspacefs.ErrInvalid) { + t.Fatal("credentialed Control constructed without credential", err) + } + // Nodes resolve with public configuration only; ownership checks follow. + binding := workspacefs.Binding{Configuration: config, Attachment: workspacefs.Attachment{Reference: workspacefs.Reference{TenantID: "33333333-3333-4333-8333-333333333333", EnvironmentID: "44444444-4444-4444-8444-444444444444", ObjectID: "55555555-5555-4555-8555-555555555555"}, + ConfigurationID: config.ID, Kind: workspacefs.AttachmentHostDirectory, Native: json.RawMessage(`{"namespace_id":"00000000-0000-4000-8000-000000000000"}`)}} + if _, err := r.Resolve(t.Context(), binding); !errors.Is(err, workspacefs.ErrOwnership) { + t.Fatal("resolver did not validate the public binding", err) + } + config.Adapter = "other" + if _, err := r.Normalize(config, nil); !errors.Is(err, workspacefs.ErrUnsupported) { + t.Fatal(err) + } + if _, err := r.Control(workspacefs.ConfigurationRecord{Configuration: config}); !errors.Is(err, workspacefs.ErrUnsupported) { + t.Fatal(err) + } +} diff --git a/services/core/internal/workspacefs/workspacefs.go b/services/core/internal/workspacefs/workspacefs.go index fd2343b36..18dacc6bd 100644 --- a/services/core/internal/workspacefs/workspacefs.go +++ b/services/core/internal/workspacefs/workspacefs.go @@ -34,12 +34,22 @@ type Reference struct { // Configuration is an immutable Core database record. Parameters belong to its // adapter; neither Core nor the public Session API interprets native paths. +// Parameters are public: they reach nodes and administrator responses. type Configuration struct { ID string `json:"id" format:"uuid" binding:"required"` Adapter string `json:"adapter" binding:"required"` Parameters json.RawMessage `json:"parameters" swaggertype:"object" binding:"required"` } +// ConfigurationRecord is a retained Configuration with its adapter-encoded +// private credential, empty when the parameters require none. Core encrypts +// Credential at rest, bound to its installation and Configuration.ID. It never +// enters a Binding, a response, an audit record or a log. +type ConfigurationRecord struct { + Configuration + Credential []byte `json:"-"` +} + type AttachmentKind string const AttachmentHostDirectory AttachmentKind = "host_directory" @@ -65,7 +75,8 @@ type Directory struct { } // Declaration describes verified behavior. CapacityQuota means the adapter -// enforces the requested capacity, not merely that it can report free space. +// enforces every positive capacity passed to Create, not merely that it can +// report free space. type Declaration struct { Attachment AttachmentKind `json:"attachment" binding:"required"` UserXAttr bool `json:"user_xattr" binding:"required"` @@ -85,8 +96,12 @@ type Control interface { Check(context.Context) error // Create retries and concurrent calls for the same immutable Reference converge // on one object without overwriting its data, including after ErrUnconfirmed. - // A terminally deleted Reference must never be recreated. - Create(context.Context, Reference) (Attachment, error) + // A terminally deleted Reference must never be recreated. capacityMiB is the + // admitted bound Core persisted with the Reference and passes unchanged on + // every retry. Zero requests no quota. A positive value requires + // Declaration.CapacityQuota; the object's allocated bytes and inodes are then + // bounded by it before Create returns, and a retry verifies that bound. + Create(ctx context.Context, reference Reference, capacityMiB uint32) (Attachment, error) // Observe returns ErrNotFound for absence, which does not settle a mutation. Observe(context.Context, Reference) (Attachment, error) // Delete retries and concurrent calls for the same Reference converge on @@ -100,6 +115,22 @@ type Resolver interface { Resolve(context.Context, Binding) (Directory, error) } +// Adapter registers one filesystem implementation. Its roles are separate and +// all mandatory. Normalize is a pure codec. Control is constructed only in Core +// from a retained record and its decrypted credential. Resolver is constructed +// from public configuration alone, so nodes never receive a credential. +// Construction performs no filesystem or network operation. +type Adapter interface { + // Normalize validates and canonicalizes public parameters and encodes the + // credential input. Empty input encodes as an empty credential. A credential + // the parameters do not require, or a missing required one, is ErrInvalid. + // An encoded credential is valid input and encodes to itself, so Core can + // re-select a retained configuration with its stored credential. + Normalize(parameters, credential json.RawMessage) (json.RawMessage, []byte, error) + Control(ConfigurationRecord) (Control, error) + Resolver(Configuration) (Resolver, error) +} + func canonicalUUID(value string) bool { id, err := uuid.Parse(value) return err == nil && id != uuid.Nil && id.String() == value diff --git a/services/core/internal/workspaces/execution.go b/services/core/internal/workspaces/execution.go index 8c1634aed..496b7bb8d 100644 --- a/services/core/internal/workspaces/execution.go +++ b/services/core/internal/workspaces/execution.go @@ -2,7 +2,9 @@ package workspaces import ( "context" + "encoding/json" "errors" + "fmt" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" @@ -11,8 +13,8 @@ import ( // Controls constructs the registered adapter for a retained configuration. type Controls interface { - Control(workspacefs.Configuration) (workspacefs.Control, error) - Normalize(workspacefs.Configuration) (workspacefs.Configuration, error) + Control(workspacefs.ConfigurationRecord) (workspacefs.Control, error) + Normalize(workspacefs.Configuration, json.RawMessage) (workspacefs.ConfigurationRecord, error) } type Ownership interface{ CheckOwnership(context.Context) error } @@ -44,7 +46,9 @@ func (e *ExecutionOperations) Declaration(ctx context.Context) (*workspacefs.Dec return &d, nil } -func validate(record Record, tenant, environment string, requirements *workspacefs.Requirements, capacity uint32, control workspacefs.Control) error { +// validate checks a retained record against its own configuration and admitted +// capacity, never today's selection or deployment request. +func validate(record Record, tenant, environment string, requirements *workspacefs.Requirements, control workspacefs.Control) error { if record.Reference.TenantID != tenant || record.Reference.EnvironmentID != environment { return workspacefs.ErrOwnership } @@ -54,13 +58,21 @@ func validate(record Record, tenant, environment string, requirements *workspace if requirements == nil { return workspacefs.ErrUnsupported } - return workspacefs.ValidateCombination(*requirements, control.Declaration(), capacity) + return workspacefs.ValidateCombination(*requirements, control.Declaration(), record.CapacityMiB) } // Ensure persists the binding before creation and retries only filesystem Create, // whose protocol requires convergence. Compute reservation follows this operation. +// capacity is the deployment's request; it is admitted only for a new binding. func (e *ExecutionOperations) Ensure(ctx context.Context, tenant, environment string, requirements *workspacefs.Requirements, capacity uint32) (*workspacefs.Binding, error) { - record, err := e.execution.Bind(ctx, tenant, environment) + record, err := e.storage.Get(ctx, tenant, environment) + switch { + case err == nil: + // Revalidate the retained binding under the Session lock. + record, err = e.execution.Bind(ctx, tenant, environment, record.Configuration.ID, record.CapacityMiB) + case errors.Is(err, ErrNotFound): + record, err = e.admit(ctx, tenant, environment, requirements, capacity) + } if errors.Is(err, ErrNotConfigured) { return nil, nil } @@ -71,7 +83,7 @@ func (e *ExecutionOperations) Ensure(ctx context.Context, tenant, environment st if err != nil { return nil, err } - if err = validate(record, tenant, environment, requirements, capacity, control); err != nil { + if err = validate(record, tenant, environment, requirements, control); err != nil { return nil, err } if record.State == Ready { @@ -83,11 +95,11 @@ func (e *ExecutionOperations) Ensure(ctx context.Context, tenant, environment st if err = e.ownership.CheckOwnership(ctx); err != nil { return nil, err } - attachment, err := control.Create(ctx, record.Reference) + attachment, err := control.Create(ctx, record.Reference, record.CapacityMiB) if err != nil { return nil, err } - if err = workspacefs.ValidateAttachment(record.Reference, record.Configuration, attachment); err != nil { + if err = workspacefs.ValidateAttachment(record.Reference, record.Configuration.Configuration, attachment); err != nil { return nil, err } if err = e.execution.MarkReady(ctx, record.Reference, attachment); err != nil { @@ -97,18 +109,39 @@ func (e *ExecutionOperations) Ensure(ctx context.Context, tenant, environment st return readyBinding(record) } +// admit validates the selected configuration against the request before any +// allocation; Bind then fences a concurrent selection change. +func (e *ExecutionOperations) admit(ctx context.Context, tenant, environment string, requirements *workspacefs.Requirements, capacity uint32) (Record, error) { + config, err := e.storage.ActiveConfiguration(ctx) + if err != nil { + return Record{}, err + } + control, err := e.controls.Control(config) + if err != nil { + return Record{}, err + } + if requirements == nil { + return Record{}, workspacefs.ErrUnsupported + } + if err = workspacefs.ValidateCombination(*requirements, control.Declaration(), capacity); err != nil { + return Record{}, err + } + return e.execution.Bind(ctx, tenant, environment, config.ID, capacity) +} + +// readyBinding projects only the public configuration; the credential stays in Core. func readyBinding(record Record) (*workspacefs.Binding, error) { if record.Attachment == nil { return nil, workspacefs.ErrOwnership } - if err := workspacefs.ValidateAttachment(record.Reference, record.Configuration, *record.Attachment); err != nil { + if err := workspacefs.ValidateAttachment(record.Reference, record.Configuration.Configuration, *record.Attachment); err != nil { return nil, err } - return &workspacefs.Binding{Configuration: record.Configuration, Attachment: *record.Attachment}, nil + return &workspacefs.Binding{Configuration: record.Configuration.Configuration, Attachment: *record.Attachment}, nil } // GetReady never creates a filesystem or substitutes today's selected adapter. -func (e *ExecutionOperations) GetReady(ctx context.Context, tenant, environment string, requirements *workspacefs.Requirements, capacity uint32) (*workspacefs.Binding, error) { +func (e *ExecutionOperations) GetReady(ctx context.Context, tenant, environment string, requirements *workspacefs.Requirements) (*workspacefs.Binding, error) { record, err := e.storage.Get(ctx, tenant, environment) if errors.Is(err, ErrNotFound) { // Existing compute without a retained binding owns its original workspace, @@ -125,7 +158,7 @@ func (e *ExecutionOperations) GetReady(ctx context.Context, tenant, environment if err != nil { return nil, err } - if err = validate(record, tenant, environment, requirements, capacity, control); err != nil { + if err = validate(record, tenant, environment, requirements, control); err != nil { return nil, err } return readyBinding(record) @@ -184,36 +217,54 @@ func (e *ExecutionOperations) delete(ctx context.Context, record Record) error { return e.execution.MarkDeleted(ctx, record.Reference) } -// Configuration reads the single operator-owned selection. -func (e *ExecutionOperations) Configuration(ctx context.Context) (workspacefs.Configuration, error) { - return e.storage.ActiveConfiguration(ctx) +// Selection reads the single operator-owned selection and tenant capacity policy. +func (e *ExecutionOperations) Selection(ctx context.Context) (Selection, error) { + return e.storage.Selection(ctx) } // Configure qualifies and publishes selection while the caller holds the shared -// deployment mutation gate. Existing bindings keep their immutable configuration. -func (e *ExecutionOperations) Configure(ctx context.Context, configuration workspacefs.Configuration, validate func(workspacefs.Declaration) error) (workspacefs.Configuration, error) { - canonical, err := e.controls.Normalize(configuration) +// deployment mutation gate. Existing bindings keep their immutable configuration +// and admitted capacity; the tenant policy applies to new bindings only. +func (e *ExecutionOperations) Configure(ctx context.Context, input SelectionInput, validate func(workspacefs.Declaration) error) (Selection, error) { + if err := input.Configuration.Validate(); err != nil { + return Selection{}, err + } + credential := input.Credential + if credential == nil { + // An encoded credential is valid credential input and encodes to itself. + existing, err := e.storage.Configuration(ctx, input.ID) + if err == nil { + credential = existing.Credential + } else if !errors.Is(err, ErrNotFound) { + return Selection{}, err + } + } + record, err := e.controls.Normalize(input.Configuration, credential) if err != nil { - return workspacefs.Configuration{}, err + return Selection{}, err } - control, err := e.controls.Control(canonical) + control, err := e.controls.Control(record) if err != nil { - return workspacefs.Configuration{}, err + return Selection{}, err + } + declaration := control.Declaration() + if err = validate(declaration); err != nil { + return Selection{}, err } - if err = validate(control.Declaration()); err != nil { - return workspacefs.Configuration{}, err + if (input.TenantCapacityMiB > 0) != declaration.CapacityQuota { + return Selection{}, fmt.Errorf("%w: tenant capacity requires, and is required by, a quota-enforcing configuration", workspacefs.ErrUnsupported) } if err = e.ownership.CheckOwnership(ctx); err != nil { - return workspacefs.Configuration{}, err + return Selection{}, err } if err = control.Check(ctx); err != nil { - return workspacefs.Configuration{}, err + return Selection{}, err } if err = e.ownership.CheckOwnership(ctx); err != nil { - return workspacefs.Configuration{}, err + return Selection{}, err } - if err = e.execution.SelectConfiguration(ctx, canonical); err != nil { - return workspacefs.Configuration{}, err + if err = e.execution.SelectConfiguration(ctx, record, input.TenantCapacityMiB); err != nil { + return Selection{}, err } - return canonical, nil + return Selection{Configuration: record.Configuration, TenantCapacityMiB: input.TenantCapacityMiB}, nil } diff --git a/services/core/internal/workspaces/execution_test.go b/services/core/internal/workspaces/execution_test.go index 7a367f017..5c04a1c6c 100644 --- a/services/core/internal/workspaces/execution_test.go +++ b/services/core/internal/workspaces/execution_test.go @@ -2,7 +2,9 @@ package workspaces import ( "context" + "encoding/json" "errors" + "strings" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" @@ -10,21 +12,33 @@ import ( ) type memoryWorkspace struct { - active workspacefs.Configuration + active workspacefs.ConfigurationRecord + retained map[string]workspacefs.ConfigurationRecord + tenantCapacity uint32 record *Record deletedSession bool allocation bool expired bool + beforeBind func() } -func (s *memoryWorkspace) ActiveConfiguration(context.Context) (workspacefs.Configuration, error) { +func (s *memoryWorkspace) ActiveConfiguration(context.Context) (workspacefs.ConfigurationRecord, error) { if s.active.ID == "" { return s.active, ErrNotConfigured } return s.active, nil } -func (s *memoryWorkspace) SelectConfiguration(_ context.Context, c workspacefs.Configuration) error { - s.active = c +func (s *memoryWorkspace) Configuration(_ context.Context, id string) (workspacefs.ConfigurationRecord, error) { + if c, ok := s.retained[id]; ok { + return c, nil + } + return workspacefs.ConfigurationRecord{}, ErrNotFound +} +func (s *memoryWorkspace) Selection(context.Context) (Selection, error) { + return Selection{Configuration: s.active.Configuration, TenantCapacityMiB: s.tenantCapacity}, nil +} +func (s *memoryWorkspace) SelectConfiguration(_ context.Context, c workspacefs.ConfigurationRecord, tenantCapacity uint32) error { + s.active, s.tenantCapacity = c, tenantCapacity return nil } func (s *memoryWorkspace) Get(context.Context, string, string) (Record, error) { @@ -33,14 +47,23 @@ func (s *memoryWorkspace) Get(context.Context, string, string) (Record, error) { } return *s.record, nil } -func (s *memoryWorkspace) Bind(_ context.Context, tenant, environment string) (Record, error) { +func (s *memoryWorkspace) Bind(_ context.Context, tenant, environment, configuration string, capacity uint32) (Record, error) { + if s.beforeBind != nil { + s.beforeBind() + } if s.record != nil { + if s.record.State == Deleting || s.record.State == Deleted { + return Record{}, ErrConflict + } return *s.record, nil } if s.active.ID == "" { return Record{}, ErrNotConfigured } - s.record = &Record{Reference: workspacefs.Reference{TenantID: tenant, EnvironmentID: environment, ObjectID: uuid.NewString()}, Configuration: s.active, State: Creating} + if s.active.ID != configuration { + return Record{}, ErrConflict + } + s.record = &Record{Reference: workspacefs.Reference{TenantID: tenant, EnvironmentID: environment, ObjectID: uuid.NewString()}, Configuration: s.active, CapacityMiB: capacity, State: Creating} return *s.record, nil } func (s *memoryWorkspace) MarkReady(_ context.Context, r workspacefs.Reference, a workspacefs.Attachment) error { @@ -71,34 +94,39 @@ func (s *memoryWorkspace) DeletionCandidates(_ context.Context, cursor string) ( type workspaceControl struct { store *memoryWorkspace + quota bool createErr, deleteErr error creates, deletes int + capacities []uint32 checks int checkErr error + normalized []json.RawMessage } -func (c *workspaceControl) Control(config workspacefs.Configuration) (workspacefs.Control, error) { +func (c *workspaceControl) Control(config workspacefs.ConfigurationRecord) (workspacefs.Control, error) { if c.store.record != nil && config.ID != c.store.record.Configuration.ID { return nil, workspacefs.ErrOwnership } return c, nil } -func (c *workspaceControl) Normalize(cn workspacefs.Configuration) (workspacefs.Configuration, error) { - return cn, nil +func (c *workspaceControl) Normalize(cn workspacefs.Configuration, credential json.RawMessage) (workspacefs.ConfigurationRecord, error) { + c.normalized = append(c.normalized, credential) + return workspacefs.ConfigurationRecord{Configuration: cn, Credential: credential}, nil } func (c *workspaceControl) Check(context.Context) error { c.checks++; return c.checkErr } func (c *workspaceControl) Declaration() workspacefs.Declaration { - return workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true} + return workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true, CapacityQuota: c.quota} } -func (c *workspaceControl) Create(_ context.Context, r workspacefs.Reference) (workspacefs.Attachment, error) { +func (c *workspaceControl) Create(_ context.Context, r workspacefs.Reference, capacity uint32) (workspacefs.Attachment, error) { c.creates++ + c.capacities = append(c.capacities, capacity) if c.store.record == nil || c.store.record.Reference != r { panic("create before durable bind") } return workspacefs.Attachment{Reference: r, ConfigurationID: c.store.record.Configuration.ID, Kind: workspacefs.AttachmentHostDirectory, Native: []byte(`{}`)}, c.createErr } func (c *workspaceControl) Observe(ctx context.Context, r workspacefs.Reference) (workspacefs.Attachment, error) { - return c.Create(ctx, r) + return c.Create(ctx, r, 0) } func (c *workspaceControl) Delete(context.Context, workspacefs.Reference) error { c.deletes++ @@ -109,7 +137,7 @@ type workspaceLease struct{ err error } func (l *workspaceLease) CheckOwnership(context.Context) error { return l.err } func fixtureWorkspace() (*ExecutionOperations, *memoryWorkspace, *workspaceControl, *workspaceLease) { - s := &memoryWorkspace{active: workspacefs.Configuration{ID: uuid.NewString(), Adapter: "fixture", Parameters: []byte(`{}`)}} + s := &memoryWorkspace{active: workspacefs.ConfigurationRecord{Configuration: workspacefs.Configuration{ID: uuid.NewString(), Adapter: "fixture", Parameters: []byte(`{}`)}}} c := &workspaceControl{store: s} l := &workspaceLease{} return NewExecution(s, s, c, l), s, c, l @@ -135,6 +163,65 @@ func TestEnsureConvergesOriginalBindingAcrossSelectionChange(t *testing.T) { t.Fatal("ready binding recreated", err) } } + +func TestEnsureValidatesBeforeBinding(t *testing.T) { + e, s, c, _ := fixtureWorkspace() + if _, err := e.Ensure(t.Context(), uuid.NewString(), uuid.NewString(), &requiredWorkspace, 1024); !errors.Is(err, workspacefs.ErrUnsupported) || s.record != nil || c.creates != 0 { + t.Fatal("unsupported request bound a workspace", err) + } + if _, err := e.Ensure(t.Context(), uuid.NewString(), uuid.NewString(), nil, 0); !errors.Is(err, workspacefs.ErrUnsupported) || s.record != nil { + t.Fatal("missing requirements bound a workspace", err) + } +} + +func TestEnsureFencesConcurrentSelectionChange(t *testing.T) { + e, s, c, _ := fixtureWorkspace() + s.beforeBind = func() { s.active.ID = uuid.NewString() } + if _, err := e.Ensure(t.Context(), uuid.NewString(), uuid.NewString(), &requiredWorkspace, 0); !errors.Is(err, ErrConflict) || s.record != nil || c.creates != 0 { + t.Fatal("binding used a configuration that was no longer selected", err) + } +} + +func TestRetainedCapacitySurvivesDeploymentChange(t *testing.T) { + e, s, c, _ := fixtureWorkspace() + c.quota = true + tenant, environment := uuid.NewString(), uuid.NewString() + c.createErr = workspacefs.ErrUnconfirmed + if _, err := e.Ensure(t.Context(), tenant, environment, &requiredWorkspace, 2048); !errors.Is(err, workspacefs.ErrUnconfirmed) || s.record.CapacityMiB != 2048 { + t.Fatal(err) + } + c.createErr = nil + if _, err := e.Ensure(t.Context(), tenant, environment, &requiredWorkspace, 4096); err != nil { + t.Fatal(err) + } + // Resume validates the retained bound against the record's configuration + // declaration, never a deployment request. + c.quota = false + if _, err := e.GetReady(t.Context(), tenant, environment, &requiredWorkspace); !errors.Is(err, workspacefs.ErrUnsupported) { + t.Fatal("bounded workspace resolved by a configuration without quota", err) + } + c.quota = true + if _, err := e.GetReady(t.Context(), tenant, environment, &requiredWorkspace); err != nil { + t.Fatal(err) + } + if len(c.capacities) != 2 || c.capacities[0] != 2048 || c.capacities[1] != 2048 { + t.Fatalf("Create capacities %v", c.capacities) + } +} + +func TestReadyBindingCarriesNoCredential(t *testing.T) { + e, s, _, _ := fixtureWorkspace() + s.active.Credential = []byte(`{"ssh_private_key":"synthetic-private-key"}`) + binding, err := e.Ensure(t.Context(), uuid.NewString(), uuid.NewString(), &requiredWorkspace, 0) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(binding) + if strings.Contains(string(raw), "synthetic-private-key") { + t.Fatalf("binding leaks credential: %s", raw) + } +} + func TestWorkspaceDeletionRequiresExplicitIntentAndReleasedCompute(t *testing.T) { e, s, c, _ := fixtureWorkspace() _, err := e.Ensure(t.Context(), uuid.NewString(), uuid.NewString(), &requiredWorkspace, 0) @@ -173,7 +260,7 @@ func TestWorkspaceDeletionAfterUnconfirmedCreationWithoutAllocation(t *testing.T func TestWorkspaceLeaseLossAndResumeNeverCreate(t *testing.T) { e, s, c, l := fixtureWorkspace() tenant, environment := uuid.NewString(), uuid.NewString() - if _, err := e.GetReady(t.Context(), tenant, environment, &requiredWorkspace, 0); err != nil || c.creates != 0 { + if _, err := e.GetReady(t.Context(), tenant, environment, &requiredWorkspace); err != nil || c.creates != 0 { t.Fatal("resume created workspace", err) } l.err = errors.New("lease lost") @@ -187,46 +274,83 @@ func TestWorkspaceLeaseLossAndResumeNeverCreate(t *testing.T) { } } -func TestGetReadyRetainsOriginalSelectionAndValidatesCapacity(t *testing.T) { +func TestGetReadyRetainsOriginalSelection(t *testing.T) { e, s, c, _ := fixtureWorkspace() tenant, environment := uuid.NewString(), uuid.NewString() - if _, err := e.Ensure(t.Context(), tenant, environment, &requiredWorkspace, 1024); !errors.Is(err, workspacefs.ErrUnsupported) || c.creates != 0 { - t.Fatal("unsupported quota created filesystem", err) - } binding, err := e.Ensure(t.Context(), tenant, environment, &requiredWorkspace, 0) if err != nil { t.Fatal(err) } s.active.ID = uuid.NewString() - resumed, err := e.GetReady(t.Context(), tenant, environment, &requiredWorkspace, 0) + resumed, err := e.GetReady(t.Context(), tenant, environment, &requiredWorkspace) if err != nil || resumed.Configuration.ID != binding.Configuration.ID || resumed.Attachment.Reference != binding.Attachment.Reference || c.creates != 1 { t.Fatal("resume changed binding", err) } - if _, err = e.GetReady(t.Context(), uuid.NewString(), environment, &requiredWorkspace, 0); !errors.Is(err, workspacefs.ErrOwnership) { + if _, err = e.GetReady(t.Context(), uuid.NewString(), environment, &requiredWorkspace); !errors.Is(err, workspacefs.ErrOwnership) { t.Fatal("foreign binding accepted", err) } } +func selectionInput(c workspacefs.Configuration, tenantCapacity uint32, credential json.RawMessage) SelectionInput { + return SelectionInput{Selection: Selection{Configuration: c, TenantCapacityMiB: tenantCapacity}, Credential: credential} +} + func TestConfigureChecksBeforeLeaseFencedSelection(t *testing.T) { e, s, c, l := fixtureWorkspace() previous := s.active.ID - next := s.active + next := s.active.Configuration next.ID = uuid.NewString() - if _, err := e.Configure(t.Context(), next, func(workspacefs.Declaration) error { return workspacefs.ErrUnsupported }); !errors.Is(err, workspacefs.ErrUnsupported) || s.active.ID != previous || c.checks != 0 { + if _, err := e.Configure(t.Context(), selectionInput(next, 0, nil), func(workspacefs.Declaration) error { return workspacefs.ErrUnsupported }); !errors.Is(err, workspacefs.ErrUnsupported) || s.active.ID != previous || c.checks != 0 { t.Fatal("unsupported selection reached native check", err) } c.checkErr = workspacefs.ErrUnavailable - if _, err := e.Configure(t.Context(), next, func(workspacefs.Declaration) error { return nil }); !errors.Is(err, workspacefs.ErrUnavailable) || s.active.ID != previous { + if _, err := e.Configure(t.Context(), selectionInput(next, 0, nil), func(workspacefs.Declaration) error { return nil }); !errors.Is(err, workspacefs.ErrUnavailable) || s.active.ID != previous { t.Fatal("failed native check published selection", err) } c.checkErr = nil l.err = errors.New("lease lost") - if _, err := e.Configure(t.Context(), next, func(workspacefs.Declaration) error { return nil }); !errors.Is(err, l.err) || s.active.ID != previous { + if _, err := e.Configure(t.Context(), selectionInput(next, 0, nil), func(workspacefs.Declaration) error { return nil }); !errors.Is(err, l.err) || s.active.ID != previous { t.Fatal("lost lease published selection", err) } l.err = nil - got, err := e.Configure(t.Context(), next, func(workspacefs.Declaration) error { return nil }) + got, err := e.Configure(t.Context(), selectionInput(next, 0, nil), func(workspacefs.Declaration) error { return nil }) if err != nil || got.ID != next.ID || s.active.ID != next.ID { t.Fatal("unconfigured Sandbox blocked filesystem selection", err) } } + +func TestConfigureTenantCapacityRequiresQuota(t *testing.T) { + e, s, c, _ := fixtureWorkspace() + next := workspacefs.Configuration{ID: uuid.NewString(), Adapter: "fixture", Parameters: []byte(`{}`)} + accept := func(workspacefs.Declaration) error { return nil } + if _, err := e.Configure(t.Context(), selectionInput(next, 4096, nil), accept); !errors.Is(err, workspacefs.ErrUnsupported) || s.active.ID == next.ID || c.checks != 0 { + t.Fatal("tenant capacity accepted without quota enforcement", err) + } + c.quota = true + if _, err := e.Configure(t.Context(), selectionInput(next, 0, nil), accept); !errors.Is(err, workspacefs.ErrUnsupported) || s.active.ID == next.ID || c.checks != 0 { + t.Fatal("quota configuration selected without tenant capacity", err) + } + got, err := e.Configure(t.Context(), selectionInput(next, 4096, nil), accept) + if err != nil || got.TenantCapacityMiB != 4096 || s.tenantCapacity != 4096 || s.active.ID != next.ID { + t.Fatal("tenant capacity not selected with configuration", err) + } +} + +func TestConfigureRetainsStoredCredentialOnlyWhenOmitted(t *testing.T) { + e, s, c, _ := fixtureWorkspace() + stored := json.RawMessage(`{"ssh_private_key":"stored"}`) + existing := workspacefs.Configuration{ID: uuid.NewString(), Adapter: "fixture", Parameters: []byte(`{}`)} + s.retained = map[string]workspacefs.ConfigurationRecord{existing.ID: {Configuration: existing, Credential: stored}} + accept := func(workspacefs.Declaration) error { return nil } + if _, err := e.Configure(t.Context(), selectionInput(existing, 0, nil), accept); err != nil || string(s.active.Credential) != string(stored) { + t.Fatal("omitted credential did not retain the stored one", err) + } + supplied := json.RawMessage(`{"ssh_private_key":"supplied"}`) + if _, err := e.Configure(t.Context(), selectionInput(existing, 0, supplied), accept); err != nil || string(c.normalized[1]) != string(supplied) { + t.Fatal("supplied credential was not normalized", err) + } + fresh := workspacefs.Configuration{ID: uuid.NewString(), Adapter: "fixture", Parameters: []byte(`{}`)} + if _, err := e.Configure(t.Context(), selectionInput(fresh, 0, nil), accept); err != nil || c.normalized[2] != nil { + t.Fatal("new configuration received a credential", err) + } +} diff --git a/services/core/internal/workspaces/storage.go b/services/core/internal/workspaces/storage.go index bfe877b1d..14d96dc1d 100644 --- a/services/core/internal/workspaces/storage.go +++ b/services/core/internal/workspaces/storage.go @@ -7,8 +7,12 @@ import ( ) // Storage reads configurations and ownership; it grants no execution authority. +// Records and configurations carry their decrypted credentials; Selection is public. type Storage interface { - ActiveConfiguration(context.Context) (workspacefs.Configuration, error) + ActiveConfiguration(context.Context) (workspacefs.ConfigurationRecord, error) + // Configuration returns a retained configuration by ID, or ErrNotFound. + Configuration(context.Context, string) (workspacefs.ConfigurationRecord, error) + Selection(context.Context) (Selection, error) Get(context.Context, string, string) (Record, error) // DeletionCandidates includes explicitly deleted Sessions even when no // compute allocation was ever made, and resumes committed deletion intent. @@ -19,12 +23,18 @@ type Storage interface { // ExecutionStorage writes only through the existing execution lease. Session // locking serializes admission and ready publication against public deletion. type ExecutionStorage interface { - // SelectConfiguration commits the selection with administrator audit on - // the execution lease, retaining immutable previous configurations. - SelectConfiguration(context.Context, workspacefs.Configuration) error - // Bind commits an identity and the selected configuration before external - // creation. Retries return that same binding across configuration changes. - Bind(context.Context, string, string) (Record, error) + // SelectConfiguration commits the selection and the tenant capacity policy + // together with administrator audit on the execution lease, retaining + // immutable previous configurations. Reusing an ID requires the same + // adapter, parameters and credential. + SelectConfiguration(context.Context, workspacefs.ConfigurationRecord, uint32) error + // Bind returns an existing binding unchanged, across configuration and + // policy changes. Otherwise it admits a new identity with the expected + // configuration ID and capacity: ErrConflict if another configuration is + // now selected; under a tenant capacity policy, ErrUnsupported for an + // unbounded request and ErrCapacityExhausted when the tenant's workspaces + // that are not yet deleted would exceed it or include an unbounded one. + Bind(ctx context.Context, tenant, environment, configuration string, capacityMiB uint32) (Record, error) MarkReady(context.Context, workspacefs.Reference, workspacefs.Attachment) error // BeginDelete requires an explicitly deleted Session and released compute. BeginDelete(context.Context, workspacefs.Reference) error diff --git a/services/core/internal/workspaces/types.go b/services/core/internal/workspaces/types.go index d952e7e59..c7ba32385 100644 --- a/services/core/internal/workspaces/types.go +++ b/services/core/internal/workspaces/types.go @@ -3,7 +3,9 @@ package workspaces import ( + "encoding/json" "errors" + "fmt" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" ) @@ -12,6 +14,9 @@ var ( ErrNotFound = errors.New("workspace not found") ErrConflict = errors.New("workspace ownership or lifecycle conflict") ErrNotConfigured = errors.New("workspace filesystem is not configured") + // ErrCapacityExhausted refuses a new workspace whose admission would exceed + // the tenant capacity policy. It is an unavailable outcome, not invalid input. + ErrCapacityExhausted = fmt.Errorf("%w: tenant workspace capacity exhausted", workspacefs.ErrUnavailable) ) type State string @@ -23,11 +28,29 @@ const ( Deleted State = "deleted" ) -// Record retains its reference and immutable configuration after deletion so -// late creation receipts cannot reintroduce a deleted workspace. +// Record retains its reference, immutable configuration and admitted capacity +// after deletion so late creation receipts cannot reintroduce a deleted +// workspace. Configuration carries the decrypted credential for Control only. type Record struct { Reference workspacefs.Reference - Configuration workspacefs.Configuration + Configuration workspacefs.ConfigurationRecord + CapacityMiB uint32 State State Attachment *workspacefs.Attachment } + +// Selection is the workspace storage runtime setting: the selected immutable +// configuration and the tenant capacity policy, written together. Zero +// TenantCapacityMiB disables tenant admission; a positive value requires a +// configuration that enforces capacity quotas. +type Selection struct { + workspacefs.Configuration + TenantCapacityMiB uint32 `json:"tenant_capacity_mib" binding:"required"` +} + +// SelectionInput adds the write-only adapter credential. Omitting it when +// re-selecting an existing configuration retains its stored credential. +type SelectionInput struct { + Selection + Credential json.RawMessage +} diff --git a/services/core/migrations/000100_workspace_capacity.sql b/services/core/migrations/000100_workspace_capacity.sql new file mode 100644 index 000000000..a8c6fdd7d --- /dev/null +++ b/services/core/migrations/000100_workspace_capacity.sql @@ -0,0 +1,68 @@ +-- +goose Up +-- Adapter-encoded private credential, sealed with the installation and +-- configuration ID. Existing configurations require none. +ALTER TABLE workspace_fs_configurations ADD COLUMN credential bytea + CHECK (credential IS NULL OR length(credential) > 0); +-- The admitted per-workspace capacity. Existing workspaces were admitted without one. +ALTER TABLE environment_workspaces ADD COLUMN capacity_mib bigint NOT NULL DEFAULT 0 + CHECK (capacity_mib BETWEEN 0 AND 4294967295); +-- The installation's single tenant capacity policy; zero disables admission. +CREATE TABLE workspace_storage_admission ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + tenant_capacity_mib bigint NOT NULL DEFAULT 0 CHECK (tenant_capacity_mib BETWEEN 0 AND 4294967295) +); +INSERT INTO workspace_storage_admission DEFAULT VALUES; + +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION immutable_workspace_configuration() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF (NEW.id, NEW.adapter, NEW.parameters, NEW.credential) IS DISTINCT FROM (OLD.id, OLD.adapter, OLD.parameters, OLD.credential) THEN + RAISE EXCEPTION 'Workspace filesystem configuration is immutable'; + END IF; + RETURN NEW; +END $$; +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION immutable_workspace_identity() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF (NEW.object_id, NEW.environment_id, NEW.configuration_id, NEW.capacity_mib) IS DISTINCT FROM (OLD.object_id, OLD.environment_id, OLD.configuration_id, OLD.capacity_mib) THEN + RAISE EXCEPTION 'Environment workspace identity is immutable'; + END IF; + RETURN NEW; +END $$; +-- +goose StatementEnd + +-- +goose Down +LOCK TABLE environment_workspaces, workspace_fs_configurations, workspace_storage_admission IN ACCESS EXCLUSIVE MODE; +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM workspace_fs_configurations WHERE credential IS NOT NULL) + OR EXISTS (SELECT 1 FROM environment_workspaces WHERE capacity_mib <> 0) + OR EXISTS (SELECT 1 FROM workspace_storage_admission WHERE tenant_capacity_mib <> 0) THEN + RAISE EXCEPTION 'Cannot discard workspace credentials, admitted capacities or tenant capacity policy'; + END IF; +END $$; +-- +goose StatementEnd +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION immutable_workspace_configuration() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF (NEW.id, NEW.adapter, NEW.parameters) IS DISTINCT FROM (OLD.id, OLD.adapter, OLD.parameters) THEN + RAISE EXCEPTION 'Workspace filesystem configuration is immutable'; + END IF; + RETURN NEW; +END $$; +-- +goose StatementEnd +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION immutable_workspace_identity() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF (NEW.object_id, NEW.environment_id, NEW.configuration_id) IS DISTINCT FROM (OLD.object_id, OLD.environment_id, OLD.configuration_id) THEN + RAISE EXCEPTION 'Environment workspace identity is immutable'; + END IF; + RETURN NEW; +END $$; +-- +goose StatementEnd +DROP TABLE workspace_storage_admission; +ALTER TABLE environment_workspaces DROP COLUMN capacity_mib; +ALTER TABLE workspace_fs_configurations DROP COLUMN credential; diff --git a/services/core/tests/integration/public_handler_fixture_test.go b/services/core/tests/integration/public_handler_fixture_test.go index 78104622a..f7d8c0a6d 100644 --- a/services/core/tests/integration/public_handler_fixture_test.go +++ b/services/core/tests/integration/public_handler_fixture_test.go @@ -28,7 +28,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/skills" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspaces" ) // testExecutorURL is the daemon URL self-hosted Sessions report unless a test @@ -277,11 +277,11 @@ func (s strictStandIn) CancelSandboxReset(context.Context, uint64) (deployment.V return deployment.View{}, nil } -func (s strictStandIn) Configuration(context.Context) (workspacefs.Configuration, error) { - s.unexpected("WorkspaceStorage.Configuration") - return workspacefs.Configuration{}, nil +func (s strictStandIn) Selection(context.Context) (workspaces.Selection, error) { + s.unexpected("WorkspaceStorage.Selection") + return workspaces.Selection{}, nil } -func (s strictStandIn) Configure(context.Context, workspacefs.Configuration) (workspacefs.Configuration, error) { +func (s strictStandIn) Configure(context.Context, workspaces.SelectionInput) (workspaces.Selection, error) { s.unexpected("WorkspaceStorage.Configure") - return workspacefs.Configuration{}, nil + return workspaces.Selection{}, nil } From 9ae66e3a1b676351d053970def8453be354d745c Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 09:11:55 +0000 Subject: [PATCH 2/3] docs: clarify deferred workspace capacity admission --- contracts/agents-api/index.md | 1 + contracts/agents-api/zh/index.md | 3 ++- docs/configuration.md | 4 +++- docs/zh/configuration.md | 6 ++++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/contracts/agents-api/index.md b/contracts/agents-api/index.md index f38f4a57e..bf37d7539 100644 --- a/contracts/agents-api/index.md +++ b/contracts/agents-api/index.md @@ -125,6 +125,7 @@ Each item is Core's deliberate or native behavior where the official service beh **Environments and Templates** +- Hosted workspace admission is limited by the [tenant capacity policy](../../docs/configuration.md#tenant-capacity) and the existing [input reservation deadline](./environments.md#reservations). - Runtimes do not enforce `disabled` or `restricted` networks, so Sessions that need them are rejected ([restricted network policy](./environments.md#restricted-network)). - `packages.system` is rejected; system packages must be preinstalled. - Cold continuation does not restore process memory, background processes or temporary system-disk changes; completed setup is not replayed. See [Environment persistence](./environments.md#runtime-capability-preparation). Cross-node memory restoration and takeover of an unconfirmed old writer are not qualified. diff --git a/contracts/agents-api/zh/index.md b/contracts/agents-api/zh/index.md index 12ba940de..c8206cad9 100644 --- a/contracts/agents-api/zh/index.md +++ b/contracts/agents-api/zh/index.md @@ -1,7 +1,7 @@ --- title: "Agents API 覆盖台账" source: contracts/agents-api/index.md -source_hash: 37355fa7100a2610ef727a3bb28f028d175be1b48b2d9f81c89b40f14cf17aef +source_hash: ddc93473d2f9acf6623072b3245b1dc244109f8fc597ef097e25d7ddf07cf392 --- Core 旨在以下方固定版本为准支持完整的 OpenAI Agents API（[public API rule](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/AGENTS.md#public-api)）。本台账记录 Core 对各项资源实现了哪些内容、哪些契约保存其详细信息，并列出相对于 OpenAI 服务的所有已知差异和所有未解决缺口。[API namespaces and credentials](../../../docs/zh/api/index.md) 说明谁调用哪些 API；[Agents API guide](../../../docs/zh/api/public-agent-api.md) 介绍使用方法。 @@ -128,6 +128,7 @@ Core 自身字段位于 `x_agents_core` 中（[Core extensions](../../../docs/zh **Environments 和 Templates** +- Hosted 工作区准入受[租户容量策略](../../../docs/zh/configuration.md#tenant-capacity)及既有[输入预留截止时间](environments.md#reservations)限制。 - Runtime 不会实施 `disabled` 或 `restricted` 网络，因此需要这些网络的 Session 会被拒绝（[restricted network policy](environments.md#restricted-network)）。 - `packages.system` 会被拒绝；系统软件包必须预先安装。 - 冷续接不恢复进程内存、后台进程或临时系统盘修改；已完成的 setup 不会重放。参见 [Environment 持久化](environments.md#runtime-capability-preparation)。跨节点内存恢复及接管未确认停止的旧写入者尚未通过资格验证。 diff --git a/docs/configuration.md b/docs/configuration.md index 55ca7ceeb..8bebb69cb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -180,7 +180,9 @@ Supported recovery for quota mode is a block-level copy of the XFS filesystem; s #### Tenant capacity -`tenant_capacity_mib` is the installation's single tenant capacity policy, written only together with the selection by `PUT /core/v1/workspace-storage` and stored in Core's database. A positive value requires a quota mode configuration, and a quota mode configuration requires a positive value. It bounds the sum of admitted capacities of one tenant's workspaces that are not yet confirmed deleted, across configuration changes; [capability validation](./workspace-provider.md#capability-validation) owns the counting rules. A refused workspace makes Session execution unavailable; deleting Sessions frees capacity once their storage is removed. Lowering the value refuses new workspaces only. +`tenant_capacity_mib` is the installation's single tenant capacity policy, written only together with the selection by `PUT /core/v1/workspace-storage` and stored in Core's database. A positive value requires a quota mode configuration, and a quota mode configuration requires a positive value. It bounds the sum of admitted capacities of one tenant's workspaces that are not yet confirmed deleted, across configuration changes; [capability validation](./workspace-provider.md#capability-validation) owns the counting rules. Lowering the value refuses new workspaces only; deleting Sessions frees capacity once their storage is removed. + +Session creation can succeed before this asynchronous workspace admission. While capacity is exhausted, Core retries without allocating the refused workspace or its compute; pending input keeps its original [Environment input deadline](../contracts/agents-api/environments.md#reservations). An idle Session created without input has no input deadline and waits for capacity. Workspaces created without quota mode remain unbounded; nothing resizes or migrates them. While one is retained, its tenant cannot admit new workspaces under a positive policy until that Session is deleted. diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index cd5792b34..bed419030 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: 2d0f6969acaa9383ef74c639547219e3146366f91a0ab5d42ad734569533c144 +source_hash: 506550dd727916c603fa2dfd713eaf190856b0536d6e4b6652a374419f2e7a4e --- Core 安装的每项设置都恰好只有一个归属位置，分属以下三类： @@ -184,7 +184,9 @@ services: #### 租户容量 {#tenant-capacity} -`tenant_capacity_mib` 是安装唯一的租户容量策略，只能与选择一起通过 `PUT /core/v1/workspace-storage` 写入，并保存在 Core 数据库中。正值要求配额模式配置，配额模式配置也要求正值。它限制单个租户所有尚未确认删除的工作区的获准容量之和，并跨配置变更生效；计数规则由[能力验证](./workspace-provider.md#capability-validation)负责。被拒绝的工作区使 Session 执行不可用；删除 Session 后，其存储被移除时即释放容量。调低该值只拒绝新工作区。 +`tenant_capacity_mib` 是安装唯一的租户容量策略，只能与选择一起通过 `PUT /core/v1/workspace-storage` 写入，并保存在 Core 数据库中。正值要求配额模式配置，配额模式配置也要求正值。它限制单个租户所有尚未确认删除的工作区的获准容量之和，并跨配置变更生效；计数规则由[能力验证](./workspace-provider.md#capability-validation)负责。调低该值只拒绝新工作区；删除 Session 后，其存储被移除时即释放容量。 + +Session 创建可以在异步工作区准入之前成功。容量耗尽时，Core 会重试，不为被拒绝的工作区分配存储或计算资源；待处理输入保留原有的 [Environment 输入截止时间](../../contracts/agents-api/zh/environments.md#reservations)。无输入创建的空闲 Session 没有输入截止时间，会等待容量释放。 非配额模式下创建的工作区保持无上限；不会调整其大小或迁移它们。只要保留此类工作区，在正策略下其租户在删除该 Session 前无法准入新工作区。 From 006437599d4ed6f707f943fa38457e8f4bae32e6 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 09:26:49 +0000 Subject: [PATCH 3/3] Release unallocated capacity after preparation failure --- contracts/agents-api/sandbox-deployment.md | 2 +- contracts/agents-api/zh/sandbox-deployment.md | 4 +- .../db/queries/runtime_lifecycle_nodes.sql | 9 +- .../internal/db/queries/runtime_nodes.sql | 5 + .../db/sqlc/runtime_lifecycle_nodes.sql.go | 25 +- .../internal/db/sqlc/runtime_nodes.sql.go | 17 + .../core/internal/deployment/allocation.go | 8 +- .../internal/deployment/allocations_test.go | 5 + .../core/internal/deployment/replacement.go | 34 +- services/core/internal/deployment/storage.go | 2 + .../internal/execution/runtime_lifecycle.go | 28 +- .../runtime_preparation_capacity_test.go | 398 ++++++++++++++++++ .../runtime_workspace_settlement_test.go | 4 +- .../postgres/deploymentpg/allocations.go | 14 +- 14 files changed, 529 insertions(+), 26 deletions(-) create mode 100644 services/core/internal/execution/runtime_preparation_capacity_test.go diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 0149d82b1..4d1aecd61 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -142,7 +142,7 @@ A node is online while it is connected under the current owner epoch with a hear Each node adds `rollout: {state, ready_generation, diagnostic?}`, where `ready_generation` is the nullable durable serving pin and `diagnostic` a fixed code for the target generation; allocation items add `deployment_generation`. Poll every five seconds only while `rollout.state` is `preparing` or `reset` is not null; old Sessions and failed, update-required or offline nodes alone do not keep polling active. -Node-backed creation validates the target deployment and commits the Session, pending Environment and any initial input without reserving compute. A full, offline or preparing fleet leaves that accepted work waiting; an unconfigured deployment, reset or unsupported combination still rejects admission. The common scheduler uses bounded rotating scans of unplaced demand, with pages ordered by recorded time and Environment ID and a fixed time boundary for each scan so continuous arrivals cannot prevent it from revisiting older work. It checks online presence, exact serving-generation readiness, address, shared capacity and the selected generation's Harness/filesystem compatibility, then prefers the newest qualifying pin. Placement is immutable once reserved until confirmed release or a settled checkpoint transfer. A bounded scan skips temporarily unavailable demand and resumes after a restart. Existing suspended allocations reserve eligible checkpoint capacity through the same lock under the [checkpoint transfer contract](../../docs/sandbox-provider.md#checkpoint-transfer); this is not a global fairness guarantee across hot restores and unplaced work. Input retains its [original five-minute deadline](./environments.md#reservations), including time waiting for capacity. +Node-backed creation validates the target deployment and commits the Session, pending Environment and any initial input without reserving compute. A full, offline or preparing fleet leaves that accepted work waiting; an unconfigured deployment, reset or unsupported combination still rejects admission. The common scheduler uses bounded rotating scans of unplaced demand, with pages ordered by recorded time and Environment ID and a fixed time boundary for each scan so continuous arrivals cannot prevent it from revisiting older work. It checks online presence, exact serving-generation readiness, address, shared capacity and the selected generation's Harness/filesystem compatibility, then prefers the newest qualifying pin. Placement is immutable once reserved until confirmed release or a settled checkpoint transfer. A bounded scan skips temporarily unavailable demand and resumes after a restart. If preparation fails before a new allocation is reserved, Core releases only that attempt’s node reservation, retaining any filesystem identity. Its next placement demand is ordered no earlier than release time, behind already-waiting work; successful retries use the newly selected generation’s specification. An allocation with an unknown compute outcome keeps its reservation. Existing suspended allocations reserve eligible checkpoint capacity through the same lock under the [checkpoint transfer contract](../../docs/sandbox-provider.md#checkpoint-transfer); this is not a global fairness guarantee across hot restores and unplaced work. Input retains its [original five-minute deadline](./environments.md#reservations), including time waiting for capacity. ## Reset diff --git a/contracts/agents-api/zh/sandbox-deployment.md b/contracts/agents-api/zh/sandbox-deployment.md index 24b678134..6c194d68b 100644 --- a/contracts/agents-api/zh/sandbox-deployment.md +++ b/contracts/agents-api/zh/sandbox-deployment.md @@ -1,7 +1,7 @@ --- title: "沙箱部署" source: contracts/agents-api/sandbox-deployment.md -source_hash: 05d854567f10ddca10d41f14423f0faa68fac4ceebca93dd1bb89129ecd6bf5c +source_hash: a806af7b78ceb630d1f2d1af0184b54967bd20f221d288e25c82965f0a1d73ef --- 沙箱部署为 Core 管理的 `openai_hosted` 执行选择 Sandbox Provider、每个沙箱的资源以及不可变的 Runtime 发行版。PostgreSQL 为每个安装维护一个当前有效选择；Web 和 Core API 写入同一配置。节点文件保存其已安装副本和特定于主机的路径，且不能覆盖其资源或 Runtime。该选择独立于 Harness。部署可以保持未配置状态，没有节点；此时它拒绝托管准入。 @@ -147,6 +147,8 @@ POST 会在持久保存候选配置之前对其进行验证，并且不会创建 节点模式创建先验证目标部署，提交 Session、pending Environment 及初始输入，但不预留计算资源。节点全部已满、离线或准备中时，已接受的工作继续等待；未配置部署、reset 或不支持的组合仍拒绝准入。共同调度器对尚未放置的需求进行有界循环扫描，分页按需求记录时间和 Environment ID 排序，每轮使用固定时间上界，避免持续的新需求阻止重新访问较早的工作。它检查在线状态、精确服务代次就绪情况、地址、共享容量和所选代次的 Harness/文件系统兼容性，然后优先选择最新的合格固定代次。预留后的 placement 保持不可变，直到确认释放或已结清的检查点转移。有界扫描跳过暂时不可调度的需求，并在重启后继续。已有 suspended allocation 按[检查点转移契约](../../../docs/zh/sandbox-provider.md#checkpoint-transfer)，通过同一容量锁预留符合条件的检查点容量；这不构成热恢复和未放置工作之间的全局公平性保证。输入保留[原始五分钟期限](./environments.md#reservations)，等待容量的时间也计入其中。 +如果准备工作在新 allocation 预留前失败，Core 仅释放该次尝试的节点预留，并保留已有文件系统身份。其下一次放置需求按不早于释放的时间排序，排在已经等待的工作之后；成功重试使用新选中代的 specification。计算结果不确定的 allocation 继续保留其预留。 + ## 重置 {#reset} 要更改后端，请启动重置： diff --git a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql index d73c69e1f..b5282a2d1 100644 --- a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql +++ b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql @@ -29,7 +29,7 @@ ORDER BY e.id LIMIT 32; -- name: GetRuntimeLifecyclePlacement :one SELECT d.provider_kind, d.mode, a.id AS allocation_id, a.node_id AS allocation_node_id, - p.node_id AS placement_node_id, p.released_at, + p.node_id AS placement_node_id, p.deployment_generation AS placement_generation, p.released_at, CASE WHEN COALESCE(a.deployment_generation, p.deployment_generation, d.generation)=d.generation THEN d.specification ELSE g.specification END::jsonb AS specification FROM environments e JOIN sessions s ON s.id=e.session_id @@ -41,10 +41,11 @@ WHERE s.tenant_id=$1 AND e.id=$2; -- name: ListPlacementDemand :many WITH demand AS ( -SELECT e.id, s.tenant_id, s.engine, (latest.id IS NOT NULL)::boolean AS retained, - CASE WHEN latest.id IS NULL THEN s.created_at - ELSE LEAST((SELECT min(r.created_at) FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()), CASE WHEN latest.compute_wake_requested THEN latest.compute_activity_at END) END::timestamptz AS demanded_at +SELECT e.id, s.tenant_id, s.engine, (latest.id IS NOT NULL OR EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id=e.id))::boolean AS retained, + GREATEST(p.released_at, CASE WHEN latest.id IS NULL THEN s.created_at + ELSE LEAST((SELECT min(r.created_at) FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()), CASE WHEN latest.compute_wake_requested THEN latest.compute_activity_at END) END)::timestamptz AS demanded_at FROM environments e JOIN sessions s ON s.id = e.session_id +LEFT JOIN runtime_placements p ON p.environment_id = e.id LEFT JOIN LATERAL (SELECT a.id, a.state, a.compute_wake_requested, a.compute_activity_at FROM runtime_allocations a WHERE a.environment_id = e.id ORDER BY a.created_at DESC, a.id DESC LIMIT 1) latest ON true WHERE s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') AND s.configuration->'environment'->>'type' = 'openai_hosted' diff --git a/services/core/internal/db/queries/runtime_nodes.sql b/services/core/internal/db/queries/runtime_nodes.sql index 6a24dbb20..c7d4b0274 100644 --- a/services/core/internal/db/queries/runtime_nodes.sql +++ b/services/core/internal/db/queries/runtime_nodes.sql @@ -90,3 +90,8 @@ UPDATE runtime_placements SET node_id = $2, deployment_generation = $3, reserved_at = clock_timestamp(), released_at = NULL WHERE runtime_placements.environment_id = $1 AND released_at IS NOT NULL AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = $1 AND a.state <> 'released'); + +-- name: ReleaseFailedRuntimePlacement :exec +UPDATE runtime_placements SET released_at = clock_timestamp() +WHERE runtime_placements.environment_id = $1 AND runtime_placements.node_id = $2 AND runtime_placements.deployment_generation = $3 AND runtime_placements.released_at IS NULL + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = $1 AND a.state <> 'released'); diff --git a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go index 0a1cb8e65..bbbb9f352 100644 --- a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go @@ -13,7 +13,7 @@ import ( const getRuntimeLifecyclePlacement = `-- name: GetRuntimeLifecyclePlacement :one SELECT d.provider_kind, d.mode, a.id AS allocation_id, a.node_id AS allocation_node_id, - p.node_id AS placement_node_id, p.released_at, + p.node_id AS placement_node_id, p.deployment_generation AS placement_generation, p.released_at, CASE WHEN COALESCE(a.deployment_generation, p.deployment_generation, d.generation)=d.generation THEN d.specification ELSE g.specification END::jsonb AS specification FROM environments e JOIN sessions s ON s.id=e.session_id @@ -30,13 +30,14 @@ type GetRuntimeLifecyclePlacementParams struct { } type GetRuntimeLifecyclePlacementRow struct { - ProviderKind string `json:"provider_kind"` - Mode string `json:"mode"` - AllocationID pgtype.UUID `json:"allocation_id"` - AllocationNodeID pgtype.UUID `json:"allocation_node_id"` - PlacementNodeID pgtype.UUID `json:"placement_node_id"` - ReleasedAt pgtype.Timestamptz `json:"released_at"` - Specification []byte `json:"specification"` + ProviderKind string `json:"provider_kind"` + Mode string `json:"mode"` + AllocationID pgtype.UUID `json:"allocation_id"` + AllocationNodeID pgtype.UUID `json:"allocation_node_id"` + PlacementNodeID pgtype.UUID `json:"placement_node_id"` + PlacementGeneration pgtype.Int8 `json:"placement_generation"` + ReleasedAt pgtype.Timestamptz `json:"released_at"` + Specification []byte `json:"specification"` } func (q *Queries) GetRuntimeLifecyclePlacement(ctx context.Context, arg GetRuntimeLifecyclePlacementParams) (GetRuntimeLifecyclePlacementRow, error) { @@ -48,6 +49,7 @@ func (q *Queries) GetRuntimeLifecyclePlacement(ctx context.Context, arg GetRunti &i.AllocationID, &i.AllocationNodeID, &i.PlacementNodeID, + &i.PlacementGeneration, &i.ReleasedAt, &i.Specification, ) @@ -56,10 +58,11 @@ func (q *Queries) GetRuntimeLifecyclePlacement(ctx context.Context, arg GetRunti const listPlacementDemand = `-- name: ListPlacementDemand :many WITH demand AS ( -SELECT e.id, s.tenant_id, s.engine, (latest.id IS NOT NULL)::boolean AS retained, - CASE WHEN latest.id IS NULL THEN s.created_at - ELSE LEAST((SELECT min(r.created_at) FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()), CASE WHEN latest.compute_wake_requested THEN latest.compute_activity_at END) END::timestamptz AS demanded_at +SELECT e.id, s.tenant_id, s.engine, (latest.id IS NOT NULL OR EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id=e.id))::boolean AS retained, + GREATEST(p.released_at, CASE WHEN latest.id IS NULL THEN s.created_at + ELSE LEAST((SELECT min(r.created_at) FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()), CASE WHEN latest.compute_wake_requested THEN latest.compute_activity_at END) END)::timestamptz AS demanded_at FROM environments e JOIN sessions s ON s.id = e.session_id +LEFT JOIN runtime_placements p ON p.environment_id = e.id LEFT JOIN LATERAL (SELECT a.id, a.state, a.compute_wake_requested, a.compute_activity_at FROM runtime_allocations a WHERE a.environment_id = e.id ORDER BY a.created_at DESC, a.id DESC LIMIT 1) latest ON true WHERE s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') AND s.configuration->'environment'->>'type' = 'openai_hosted' diff --git a/services/core/internal/db/sqlc/runtime_nodes.sql.go b/services/core/internal/db/sqlc/runtime_nodes.sql.go index e28353bcf..3c1306a11 100644 --- a/services/core/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_nodes.sql.go @@ -492,6 +492,23 @@ func (q *Queries) LockRuntimeNodePresence(ctx context.Context, id pgtype.UUID) ( return id, err } +const releaseFailedRuntimePlacement = `-- name: ReleaseFailedRuntimePlacement :exec +UPDATE runtime_placements SET released_at = clock_timestamp() +WHERE runtime_placements.environment_id = $1 AND runtime_placements.node_id = $2 AND runtime_placements.deployment_generation = $3 AND runtime_placements.released_at IS NULL + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = $1 AND a.state <> 'released') +` + +type ReleaseFailedRuntimePlacementParams struct { + EnvironmentID pgtype.UUID `json:"environment_id"` + NodeID pgtype.UUID `json:"node_id"` + DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` +} + +func (q *Queries) ReleaseFailedRuntimePlacement(ctx context.Context, arg ReleaseFailedRuntimePlacementParams) error { + _, err := q.db.Exec(ctx, releaseFailedRuntimePlacement, arg.EnvironmentID, arg.NodeID, arg.DeploymentGeneration) + return err +} + const releaseRuntimePlacement = `-- name: ReleaseRuntimePlacement :exec UPDATE runtime_placements SET released_at=COALESCE(released_at,clock_timestamp()) WHERE environment_id=$1 ` diff --git a/services/core/internal/deployment/allocation.go b/services/core/internal/deployment/allocation.go index 50909468e..d578e4269 100644 --- a/services/core/internal/deployment/allocation.go +++ b/services/core/internal/deployment/allocation.go @@ -116,7 +116,8 @@ type UnallocatedEnvironment struct{ ID, TenantID string } // belongs to Session creation or its oldest pending input, not to a retry. type PlacementDemand struct { UnallocatedEnvironment - Engine string + Engine string + // Retained requires external storage because an allocation or filesystem identity already exists. Retained bool At time.Time } @@ -142,8 +143,9 @@ type LifecyclePlacement struct { // its node. AllocationID, AllocationNodeID string // PlacementNodeID is empty without a placement. - PlacementNodeID string - PlacementReleased bool + PlacementNodeID string + PlacementReleased bool + PlacementGeneration uint64 } // NodeAllocation is an unreleased allocation a node serves. diff --git a/services/core/internal/deployment/allocations_test.go b/services/core/internal/deployment/allocations_test.go index 22d110171..7bad4ffa3 100644 --- a/services/core/internal/deployment/allocations_test.go +++ b/services/core/internal/deployment/allocations_test.go @@ -698,3 +698,8 @@ func TestBeginRestoreDirectKeepsAllocationAndRetention(t *testing.T) { t.Fatal("restore bypassed capacity admission", err) } } + +func (f *fakeReservationTx) ReleaseFailedPlacement(placement.Reserved) error { + unexpected(f.t, "ReleaseFailedPlacement") + return nil +} diff --git a/services/core/internal/deployment/replacement.go b/services/core/internal/deployment/replacement.go index 980ba5bd2..d97ec2cf9 100644 --- a/services/core/internal/deployment/replacement.go +++ b/services/core/internal/deployment/replacement.go @@ -77,7 +77,7 @@ func (e *ExecutionOperations) EnsurePlacement(ctx context.Context, key Allocatio if err != nil { return err } - filtered, err := e.eligibleNodes(d, nodes, locked.Engine, found, tx.LoadGenerationSpecification) + filtered, err := e.eligibleNodes(d, nodes, locked.Engine, found || environment.ExternalWorkspace, tx.LoadGenerationSpecification) if err != nil { return err } @@ -138,3 +138,35 @@ func (e *ExecutionOperations) eligibleNodes(d placement.Deployment, nodes []plac } return filtered, nil } + +// ReleaseFailedPlacement yields a failed preparation attempt without touching +// filesystem identity or an allocation that already owns compute. +func (e *ExecutionOperations) ReleaseFailedPlacement(ctx context.Context, key AllocationKey, expected placement.Reserved) error { + key, err := key.parse() + if err != nil { + return err + } + if expected.NodeID == "" || expected.Generation == 0 { + return sandbox.ErrInvalid + } + return e.storage.WithReservation(ctx, key, func(_ sessions.LockedSession, tx ReservationTx) error { + if _, err := tx.LockDeployment(); err != nil { + return err + } + existing, found, err := tx.FindAllocation() + if err != nil { + return err + } + if found && existing.State != "released" { + return nil + } + reserved, err := tx.LoadReserved() + if err != nil { + return err + } + if reserved.Released || reserved.NodeID != expected.NodeID || reserved.Generation != expected.Generation { + return nil + } + return tx.ReleaseFailedPlacement(expected) + }) +} diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index 5277a9c3a..163daa263 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -104,6 +104,8 @@ type ReservationTx interface { LoadNodes() ([]placement.Node, error) // ReservePlacement inserts a first reservation or replaces a released reservation without changing historical receipts. ReservePlacement(placement.Placement) error + // ReleaseFailedPlacement yields only the exact reservation without active compute. + ReleaseFailedPlacement(placement.Reserved) error } // AllocationTx is one Session-locked allocation change. Each write applies to diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 278d30776..560de108b 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -12,6 +12,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -180,11 +181,36 @@ func (w *Worker) ProvisionEnvironment(ctx context.Context, tenant, environment, } // provision runs under the lifecycle gate and uses the durable one-shot receipt. -func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, providerKey string) (deployment.Allocation, error) { +func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, providerKey string) (result deployment.Allocation, resultErr error) { provider := r.config.Provider if providerKey != r.config.InstallationID { return deployment.Allocation{}, sandbox.ErrInvalid } + + if r.nodeID != "" { + key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment} + target, err := r.reader.LifecyclePlacement(ctx, key) + if err != nil { + return deployment.Allocation{}, err + } + if target.PlacementNodeID != r.nodeID { + return deployment.Allocation{}, sandbox.ErrOwnership + } + if target.PlacementReleased && target.AllocationID == "" { + return deployment.Allocation{}, deployment.ErrAllocationConflict + } + if !target.PlacementReleased { + expected := placement.Reserved{NodeID: target.PlacementNodeID, Generation: target.PlacementGeneration} + defer func() { + if resultErr == nil || result.ID != "" { + return + } + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + resultErr = errors.Join(resultErr, r.deployment.ReleaseFailedPlacement(cleanup, key, expected)) + }() + } + } environmentValue, err := r.sessions.GetEnvironment(ctx, tenant, environment) if err != nil { return deployment.Allocation{}, err diff --git a/services/core/internal/execution/runtime_preparation_capacity_test.go b/services/core/internal/execution/runtime_preparation_capacity_test.go new file mode 100644 index 000000000..9e19a286b --- /dev/null +++ b/services/core/internal/execution/runtime_preparation_capacity_test.go @@ -0,0 +1,398 @@ +package execution + +import ( + "context" + "encoding/json" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" + "strings" + "sync" + "testing" + + v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspaces" + "github.com/google/uuid" +) + +type boundedSettlementControl struct{ *settlementWorkspaceControl } + +func (c boundedSettlementControl) Declaration() workspacefs.Declaration { + d := c.settlementWorkspaceControl.Declaration() + d.CapacityQuota = true + return d +} + +type boundedSettlementControls struct{ control boundedSettlementControl } + +func (c boundedSettlementControls) Control(workspacefs.ConfigurationRecord) (workspacefs.Control, error) { + return c.control, nil +} +func (c boundedSettlementControls) Normalize(config workspacefs.Configuration, credential json.RawMessage) (workspacefs.ConfigurationRecord, error) { + return workspacefs.ConfigurationRecord{Configuration: config, Credential: credential}, nil +} + +func TestWorkspaceQuotaRefusalYieldsComputeToAnotherTenant(t *testing.T) { + provider := &retentionProvider{} + f := newWorkspaceSettlementFixture(t, provider) + r, a := f.lifecycle, f.session + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_deployment SET specification=jsonb_set(jsonb_set(specification,'{resources,environment_disk_mib}','1024'),'{workspace,capacity_quota}','true')`); err != nil { + t.Fatal(err) + } + writer := f.writer + r.workspaces = workspaces.NewExecution(f.storage, writer, boundedSettlementControls{boundedSettlementControl{f.control}}, r.lease) + spec, err := r.workspaceSpecification(t.Context(), deployment.AllocationKey{TenantID: a.TenantID, EnvironmentID: a.Environment.ID}, "") + if err != nil { + t.Fatal(err) + } + capacity := spec.Resources.EnvironmentDiskMiB + if capacity == 0 { + t.Fatal("fixture requires bounded workspace") + } + if _, err := f.pool.Exec(t.Context(), `UPDATE workspace_storage_admission SET tenant_capacity_mib=$1`, capacity); err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET max_active=1,max_retained=3 WHERE id=$1`, r.nodeID); err != nil { + t.Fatal(err) + } + // An independent retained object consumes A's tenant storage, without compute. + storageSession, storageEnvironment := uuid.NewString(), uuid.NewString() + if _, err := f.pool.Exec(t.Context(), `INSERT INTO sessions(id,tenant_id,engine,idempotency_key,request_hash,configuration) VALUES($1,$2,'codex',$3,'fixture','{"environment":{"type":"openai_hosted"}}')`, storageSession, a.TenantID, uuid.NewString()); err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), `INSERT INTO environments(id,session_id) VALUES($1,$2)`, storageEnvironment, storageSession); err != nil { + t.Fatal(err) + } + occupied, err := writer.Bind(t.Context(), a.TenantID, storageEnvironment, f.control.configuration.ID, capacity) + if err != nil { + t.Fatal(err) + } + created, err := f.sessions.CreateSession(t.Context(), uuid.NewString(), sessions.CreateSession{SupportsRetainedNativeHistory: true, Creator: identity.Subject{Kind: "service_account", ID: "fixture"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}`), ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://model.fixture.example/v1", APIKey: "fixture-key"}, ModelProviderSource: v1.ExecutionSourceSession}) + if err != nil { + t.Fatal(err) + } + b := created.Session + if _, err := f.pool.Exec(t.Context(), `UPDATE environments SET status='failed' WHERE id=$1`, storageEnvironment); err != nil { + t.Fatal(err) + } + _, err = r.provision(t.Context(), a.TenantID, a.Environment.ID, r.config.InstallationID) + if !errors.Is(err, workspaces.ErrCapacityExhausted) { + t.Fatal("not the real quota refusal", err) + } + nodes, err := r.reader.Nodes(t.Context()) + if err != nil { + t.Fatal(err) + } + if len(nodes) != 1 || nodes[0].Active != 0 || provider.creates != 0 { + t.Fatalf("quota refusal retained compute capacity without a VM: nodes=%+v creates=%d", nodes, provider.creates) + } + r.config.Mode = string(sandbox.DeploymentNodes) + manager := &runtimeManager{ctx: t.Context(), config: r.config, nodes: map[string]*runtimeNode{r.nodeID: {lifecycle: r}}, workspaces: r.workspaces, sessions: r.sessions, sessionExecution: r.sessionExecution, deployment: r.deployment, deploymentService: r.deployments, deploymentReader: r.reader, lease: r.lease, registry: r.registry} + for range 3 { + if err := manager.reservePlacements(t.Context()); err != nil { + t.Fatal(err) + } + if err := r.provisionPending(t.Context()); err != nil { + t.Fatal(err) + } + } + allocated, err := r.reader.EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: b.TenantID, EnvironmentID: b.Environment.ID}) + if err != nil || allocated.State != "running" || provider.creates != 1 { + t.Fatal("quota refused A starved admissible B", allocated, err, provider.creates) + } + if err := f.sessions.DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: a.TenantID, SessionID: storageSession}); err != nil { + t.Fatal(err) + } + if err := writer.BeginDelete(t.Context(), occupied.Reference); err != nil { + t.Fatal(err) + } + if err := writer.MarkDeleted(t.Context(), occupied.Reference); err != nil { + t.Fatal(err) + } + if err := f.sessions.DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: b.TenantID, SessionID: b.ID}); err != nil { + t.Fatal(err) + } + allocated, err = r.deployment.RequestCleanup(t.Context(), allocated) + if err != nil { + t.Fatal(err) + } + if err := provider.KillCompute(t.Context(), runtimeReference(allocated), sandbox.Compute{}); err != nil { + t.Fatal(err) + } + if _, err := r.deployment.ReleaseAllocation(t.Context(), allocated); err != nil { + t.Fatal(err) + } + for range 3 { + if err := manager.reservePlacements(t.Context()); err != nil { + t.Fatal(err) + } + if err := r.provisionPending(t.Context()); err != nil { + t.Fatal(err) + } + } + recovered, err := r.reader.EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: a.TenantID, EnvironmentID: a.Environment.ID}) + if err != nil || recovered.State != "running" || provider.creates != 2 { + t.Fatal("storage release did not recover A", recovered, err, provider.creates) + } +} + +// An unsettled filesystem create keeps its durable identity while yielding +// compute. In contrast, an unsettled compute create already owns its slot. +type unsettledCapacityControl struct { + boundedSettlementControl + refs []workspacefs.Reference +} + +func (c *unsettledCapacityControl) Create(ctx context.Context, ref workspacefs.Reference, capacity uint32) (workspacefs.Attachment, error) { + c.refs = append(c.refs, ref) + if len(c.refs) == 1 { + return workspacefs.Attachment{}, workspacefs.ErrUnconfirmed + } + return c.settlementWorkspaceControl.Create(ctx, ref, capacity) +} + +type unsettledCapacityControls struct { + boundedSettlementControls + control *unsettledCapacityControl +} + +func (c unsettledCapacityControls) Control(workspacefs.ConfigurationRecord) (workspacefs.Control, error) { + return c.control, nil +} +func TestUnsettledFilesystemPreparationYieldsOnlyCompute(t *testing.T) { + provider := &retentionProvider{} + f := newWorkspaceSettlementFixture(t, provider) + r, s := f.lifecycle, f.session + control := &unsettledCapacityControl{boundedSettlementControl: boundedSettlementControl{f.control}} + r.workspaces = workspaces.NewExecution(f.storage, f.writer, unsettledCapacityControls{control: control}, r.lease) + key := deployment.AllocationKey{TenantID: s.TenantID, EnvironmentID: s.Environment.ID} + if _, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID); !errors.Is(err, workspacefs.ErrUnconfirmed) { + t.Fatal(err) + } + record, err := f.storage.Get(t.Context(), s.TenantID, s.Environment.ID) + if err != nil || record.State != workspaces.Creating || provider.creates != 0 { + t.Fatal(record, err, provider.creates) + } + target, err := r.reader.LifecyclePlacement(t.Context(), key) + if err != nil || !target.PlacementReleased || target.AllocationID != "" { + t.Fatal(target, err) + } + // A newer generation without external storage must not displace this + // unallocated Session's already-bound filesystem onto owned ephemeral disk. + setup, err := r.deployments.Setup(t.Context()) + if err != nil { + t.Fatal(err) + } + updated := setup.Specification + updated.Workspace = nil + updated.Resources.EnvironmentDiskMiB = 8192 + view, err := r.deployment.Update(adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", RequestID: uuid.NewString(), TraceID: uuid.NewString()}), r.config.InstallationID, sandbox.Selection{Provider: setup.Provider, ExpectedGeneration: setup.Generation, DeploymentSpec: updated}) + if err != nil { + t.Fatal(err) + } + token, err := r.deployments.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + if err != nil { + t.Fatal(err) + } + newer := deployment.Enrollment{NodeID: uuid.NewString(), Name: "newer owned workspace", Credential: strings.Repeat("n", 64), Provider: view.Provider, BackendFingerprint: strings.Repeat("c", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest, CoreURL: fixturePublicURL} + if _, err := r.deployments.Enroll(t.Context(), token.Token, newer); err != nil { + t.Fatal(err) + } + epoch, err := r.reader.OwnerEpoch(t.Context()) + if err != nil { + t.Fatal(err) + } + connection := uuid.NewString() + if err := r.deployments.ConnectNode(t.Context(), newer.NodeID, connection, epoch); err != nil { + t.Fatal(err) + } + if err := r.deployments.Heartbeat(t.Context(), newer.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}}); err != nil { + t.Fatal(err) + } + demands, _, err := r.reader.PlacementDemand(t.Context(), deployment.PlacementDemandCursor{}) + if err != nil || len(demands) != 1 || !demands[0].Retained { + t.Fatal("bound FS omitted from pressure eligibility", demands, err) + } + selected, err := r.deployment.EnsurePlacement(t.Context(), key, r.config.InstallationID) + if err != nil || selected.NodeID != r.nodeID || selected.Generation != setup.Generation { + t.Fatal("selected generation cannot attach retained filesystem", selected, err) + } + if _, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID); err != nil { + t.Fatal(err) + } + if len(control.refs) != 2 || control.refs[0] != control.refs[1] || control.refs[0] != record.Reference || provider.creates != 1 { + t.Fatal("retry replaced filesystem identity or duplicated compute", control.refs, provider.creates) + } +} +func TestPreparationReleaseFencesPlacementAndAllocation(t *testing.T) { + f := newWorkspaceSettlementFixture(t, &unknownWorkspaceCreation{}) + r, s := f.lifecycle, f.session + key := deployment.AllocationKey{TenantID: s.TenantID, EnvironmentID: s.Environment.ID} + target, err := r.reader.LifecyclePlacement(t.Context(), key) + if err != nil { + t.Fatal(err) + } + expected := placement.Reserved{NodeID: target.PlacementNodeID, Generation: target.PlacementGeneration} + for _, stale := range []placement.Reserved{{NodeID: uuid.NewString(), Generation: expected.Generation}, {NodeID: expected.NodeID, Generation: expected.Generation + 1}} { + if err := r.deployment.ReleaseFailedPlacement(t.Context(), key, stale); err != nil { + t.Fatal(err) + } + current, err := r.reader.LifecyclePlacement(t.Context(), key) + if err != nil || current.PlacementReleased { + t.Fatal("stale attempt released newer placement", current, err) + } + } + owner, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) + if !errors.Is(err, sandbox.ErrComputeUnconfirmed) || owner.ID == "" { + t.Fatal(owner, err) + } + if err := r.deployment.ReleaseFailedPlacement(t.Context(), key, expected); err != nil { + t.Fatal(err) + } + current, err := r.reader.LifecyclePlacement(t.Context(), key) + if err != nil || current.PlacementReleased || current.AllocationID != owner.ID { + t.Fatal("unknown compute lost its slot", current, err) + } +} + +func TestPreparationReleaseSerializesWithAllocationAndTerminalSession(t *testing.T) { + for _, action := range []string{"allocation", "archive", "delete"} { + t.Run(action, func(t *testing.T) { + f := newWorkspaceSettlementFixture(t, &retentionProvider{}) + r, s := f.lifecycle, f.session + key := deployment.AllocationKey{TenantID: s.TenantID, EnvironmentID: s.Environment.ID} + target, err := r.reader.LifecyclePlacement(t.Context(), key) + if err != nil { + t.Fatal(err) + } + expected := placement.Reserved{NodeID: target.PlacementNodeID, Generation: target.PlacementGeneration} + var wg sync.WaitGroup + var releaseErr, actionErr error + wg.Go(func() { releaseErr = r.deployment.ReleaseFailedPlacement(t.Context(), key, expected) }) + wg.Go(func() { + switch action { + case "allocation": + _, actionErr = r.deployment.ReserveAllocation(t.Context(), key, r.config.InstallationID, runtimedevice.HashCredential("fixture")) + case "archive": + _, actionErr = r.deployment.ArchiveSession(adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", ProjectID: f.projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}), s.TenantID, s.ID, expected.Generation) + case "delete": + actionErr = f.sessions.DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: s.TenantID, SessionID: s.ID}) + } + }) + wg.Wait() + if releaseErr != nil { + t.Fatal(releaseErr) + } + current, err := r.reader.LifecyclePlacement(t.Context(), key) + if err != nil { + t.Fatal(err) + } + if action == "allocation" { + if actionErr == nil && (current.AllocationID == "" || current.PlacementReleased) { + t.Fatal("allocated compute lost slot", current) + } + if actionErr != nil && current.AllocationID != "" { + t.Fatal("failed allocation persisted", current, actionErr) + } + } else { + if actionErr != nil { + t.Fatal(actionErr) + } + if !current.PlacementReleased { + t.Fatal("terminal session retained empty slot", current) + } + if _, err := r.deployment.EnsurePlacement(t.Context(), key, r.config.InstallationID); err == nil { + t.Fatal("terminal session reacquired capacity") + } + } + }) + } +} + +func TestPreparationRejectsStaleNodeLaneBeforeAllocation(t *testing.T) { + provider := &retentionProvider{} + f := newWorkspaceSettlementFixture(t, provider) + r, s := f.lifecycle, f.session + r.nodeID = uuid.NewString() // A caller routed before reassignment now holds an obsolete lane. + _, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) + if !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal(err) + } + if _, err := r.reader.EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: s.TenantID, EnvironmentID: s.Environment.ID}); !errors.Is(err, deployment.ErrNotFound) { + t.Fatal("stale lane reserved undelivered compute", err) + } + if provider.creates != 0 { + t.Fatal(provider.creates) + } +} + +type unavailableCapacityControls struct{ settlementWorkspaceControls } + +func (c unavailableCapacityControls) Control(workspacefs.ConfigurationRecord) (workspacefs.Control, error) { + return nil, workspacefs.ErrUnavailable +} +func TestRetainedPreparationFailureYieldsToWaitingDemand(t *testing.T) { + f := newWorkspaceSettlementFixture(t, &retentionProvider{}) + r, s := f.lifecycle, f.session + owner, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + // Seed an already-qualified completion and expired checkpoint; the ordinary + // lifecycle must settle and release it before cold replacement is eligible. + for _, q := range []struct { + sql string + id string + }{ + {`UPDATE environments SET initialization='complete',status='disconnected' WHERE id=$1`, owner.EnvironmentID}, + {`UPDATE devices SET supported_agent_kinds='[{"kind":"codex","available":true,"capabilities":{"retained_native_history":true}}]' WHERE id=$1`, owner.DeviceID}, + {`UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"current":{"id":"old-compute"},"snapshot":{"id":"old-snapshot","compatibility":{"artifact_domain":"fixture-store","execution_class":"fixture-runtime"}}}',compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID}, + } { + if _, err := f.pool.Exec(t.Context(), q.sql, q.id); err != nil { + t.Fatal(err) + } + } + owner, err = r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + if err := r.observe(t.Context(), owner); err != nil { + t.Fatal(err) + } + input, err := f.sessions.ReserveEnvironmentInput(t.Context(), s.TenantID, s.ID, "retry-input", []sessions.Input{{Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":"resume"}]}]}`)}}) + if err != nil { + t.Fatal(err) + } + b, err := f.sessions.CreateSession(t.Context(), s.TenantID, sessions.CreateSession{SupportsRetainedNativeHistory: true, Creator: identity.Subject{Kind: "service_account", ID: "fixture"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: s.Configuration, ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://model.fixture.example/v1", APIKey: "fixture-key"}, ModelProviderSource: v1.ExecutionSourceSession}) + if err != nil { + t.Fatal(err) + } + if _, err := r.deployment.EnsurePlacement(t.Context(), owner.Key(), r.config.InstallationID); err != nil { + t.Fatal(err) + } + filesystems := r.workspaces + r.workspaces = workspaces.NewExecution(f.storage, f.writer, unavailableCapacityControls{}, r.lease) + if _, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID); !errors.Is(err, workspacefs.ErrUnavailable) { + t.Fatal(err) + } + r.workspaces = filesystems + r.config.Mode = string(sandbox.DeploymentNodes) + manager := &runtimeManager{ctx: t.Context(), config: r.config, nodes: map[string]*runtimeNode{r.nodeID: {lifecycle: r}}, deployment: r.deployment, deploymentReader: r.reader, lease: r.lease} + if err := manager.reservePlacements(t.Context()); err != nil { + t.Fatal(err) + } + placed, err := r.reader.LifecyclePlacement(t.Context(), deployment.AllocationKey{TenantID: b.Session.TenantID, EnvironmentID: b.Session.Environment.ID}) + if err != nil || placed.PlacementNodeID != r.nodeID || placed.PlacementReleased { + t.Fatal("retained retry starved waiting demand", placed, err) + } + pending, err := r.sessions.GetEnvironmentInputReservation(t.Context(), s.TenantID, s.ID, input.ID) + if err != nil || !pending.Deadline.Equal(input.Deadline) || pending.State != sessions.EnvironmentInputPending { + t.Fatal("yield changed original input deadline", pending, err) + } +} diff --git a/services/core/internal/execution/runtime_workspace_settlement_test.go b/services/core/internal/execution/runtime_workspace_settlement_test.go index e6908caee..13ad9efd2 100644 --- a/services/core/internal/execution/runtime_workspace_settlement_test.go +++ b/services/core/internal/execution/runtime_workspace_settlement_test.go @@ -67,8 +67,10 @@ type workspaceSettlementFixture struct { lifecycle *runtimeLifecycle sessions *sessions.Service storage *workspacepg.Store + writer *workspacepg.Execution control *settlementWorkspaceControl session sessions.Session + projectID string } func newWorkspaceSettlementFixture(t *testing.T, provider sandbox.SandboxProvider) workspaceSettlementFixture { @@ -124,7 +126,7 @@ func workspaceSettlementFixtureWithSetup(t *testing.T, provider sandbox.SandboxP filesystems := workspaces.NewExecution(storage, writer, settlementWorkspaceControls{control}, lease) declaration := control.Declaration() lifecycle := &runtimeLifecycle{nodeID: nodeID, registry: runtimegateway.NewRegistry(), sessions: sessionReader, sessionExecution: owner.Sessions, deployment: operations, deployments: deployments, reader: reader, lease: lease, workspaces: filesystems, config: RuntimeProvider{InstallationID: installation, Provider: provider, Workspace: &declaration, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} - return workspaceSettlementFixture{pool: pool, lifecycle: lifecycle, sessions: service, storage: storage, control: control, session: session} + return workspaceSettlementFixture{pool: pool, lifecycle: lifecycle, sessions: service, storage: storage, writer: writer, control: control, session: session, projectID: project.ID} } func TestSettledWorkspaceRefusalRetainsReceiptUntilExplicitSessionDeletion(t *testing.T) { diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index 7d67fff4b..6ae6d1871 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -252,11 +252,11 @@ func (t *reservationTx) LoadNodes() ([]placement.Node, error) { } func (t *reservationTx) ReservePlacement(p placement.Placement) error { - _, found, err := t.FindAllocation() + reserved, err := t.LoadReserved() if err != nil { return err } - if !found { + if reserved.NodeID == "" { return placementpg.ReservePlacement(t.ctx, t.q, t.session, p) } node, err := parseID(p.NodeID) @@ -646,7 +646,7 @@ func (s *Store) LifecyclePlacement(ctx context.Context, key deployment.Allocatio return deployment.LifecyclePlacement{}, err } return deployment.LifecyclePlacement{Specification: row.Specification, Provider: row.ProviderKind, Mode: row.Mode, AllocationID: uuidString(row.AllocationID), AllocationNodeID: uuidString(row.AllocationNodeID), - PlacementNodeID: uuidString(row.PlacementNodeID), PlacementReleased: row.ReleasedAt.Valid}, nil + PlacementNodeID: uuidString(row.PlacementNodeID), PlacementGeneration: uint64(row.PlacementGeneration.Int64), PlacementReleased: row.ReleasedAt.Valid}, nil } func (s *Store) Activity(ctx context.Context, allocationID string) (deployment.Activity, error) { @@ -660,3 +660,11 @@ func (s *Store) Activity(ctx context.Context, allocationID string) (deployment.A func (t *reservationTx) LoadGenerationSpecification(generation uint64) (deployment.GenerationSpecification, error) { return (unit{ctx: t.ctx, q: t.q}).LoadGenerationSpecification(generation) } + +func (t *reservationTx) ReleaseFailedPlacement(p placement.Reserved) error { + node, err := parseID(p.NodeID) + if err != nil { + return err + } + return t.q.ReleaseFailedRuntimePlacement(t.ctx, sqlc.ReleaseFailedRuntimePlacementParams{EnvironmentID: t.environment, NodeID: node, DeploymentGeneration: pgtype.Int8{Int64: int64(p.Generation), Valid: true}}) +}